2024-11-17 08:22:47,949 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-17 08:22:47,980 main DEBUG Took 0.027230 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 08:22:47,981 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 08:22:47,981 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 08:22:47,983 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 08:22:47,984 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,000 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 08:22:48,017 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,020 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,022 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,024 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,025 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,027 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,029 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,029 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,030 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,031 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,032 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,033 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,034 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,035 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:22:48,036 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,037 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 08:22:48,039 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:22:48,041 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 08:22:48,044 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 08:22:48,045 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 08:22:48,046 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 08:22:48,047 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 08:22:48,060 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 08:22:48,063 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 08:22:48,065 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 08:22:48,066 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 08:22:48,066 main DEBUG createAppenders(={Console}) 2024-11-17 08:22:48,067 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-17 08:22:48,068 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-17 08:22:48,068 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-17 08:22:48,069 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 08:22:48,069 main DEBUG OutputStream closed 2024-11-17 08:22:48,069 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 08:22:48,070 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 08:22:48,070 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2dc54ad4 OK 2024-11-17 08:22:48,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 08:22:48,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 08:22:48,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 08:22:48,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 08:22:48,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 08:22:48,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 08:22:48,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 08:22:48,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 08:22:48,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 08:22:48,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 08:22:48,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 08:22:48,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 08:22:48,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 08:22:48,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 08:22:48,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 08:22:48,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 08:22:48,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 08:22:48,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 08:22:48,185 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 08:22:48,186 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4f9a2c08) with optional ClassLoader: null 2024-11-17 08:22:48,186 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 08:22:48,187 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4f9a2c08] started OK. 2024-11-17T08:22:48,564 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a 2024-11-17 08:22:48,568 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 08:22:48,568 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T08:22:48,607 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStreamFSHLog timeout: 26 mins 2024-11-17T08:22:48,631 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767, deleteOnExit=true 2024-11-17T08:22:48,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/test.cache.data in system properties and HBase conf 2024-11-17T08:22:48,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T08:22:48,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir in system properties and HBase conf 2024-11-17T08:22:48,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T08:22:48,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T08:22:48,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T08:22:48,749 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T08:22:48,869 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T08:22:48,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T08:22:48,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T08:22:48,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T08:22:48,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T08:22:48,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T08:22:48,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T08:22:48,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T08:22:48,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T08:22:48,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T08:22:48,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/nfs.dump.dir in system properties and HBase conf 2024-11-17T08:22:48,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/java.io.tmpdir in system properties and HBase conf 2024-11-17T08:22:48,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T08:22:48,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T08:22:48,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T08:22:50,041 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T08:22:50,153 INFO [Time-limited test {}] log.Log(170): Logging initialized @3216ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T08:22:50,238 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T08:22:50,318 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T08:22:50,350 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T08:22:50,350 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T08:22:50,352 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T08:22:50,372 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T08:22:50,378 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c2d1933{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir/,AVAILABLE} 2024-11-17T08:22:50,380 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a9b22e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T08:22:50,587 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44567ea4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/java.io.tmpdir/jetty-localhost-41369-hadoop-hdfs-3_4_1-tests_jar-_-any-5865817692316965387/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T08:22:50,597 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@216da5e5{HTTP/1.1, (http/1.1)}{localhost:41369} 2024-11-17T08:22:50,597 INFO [Time-limited test {}] server.Server(415): Started @3661ms 2024-11-17T08:22:51,230 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T08:22:51,239 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T08:22:51,240 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T08:22:51,240 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T08:22:51,240 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T08:22:51,243 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b53287d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir/,AVAILABLE} 2024-11-17T08:22:51,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ca42ed7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T08:22:51,358 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7ec85977{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/java.io.tmpdir/jetty-localhost-36463-hadoop-hdfs-3_4_1-tests_jar-_-any-8886498461816612212/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T08:22:51,359 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e913c20{HTTP/1.1, (http/1.1)}{localhost:36463} 2024-11-17T08:22:51,359 INFO [Time-limited test {}] server.Server(415): Started @4423ms 2024-11-17T08:22:51,416 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T08:22:51,603 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T08:22:51,610 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T08:22:51,612 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T08:22:51,612 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T08:22:51,612 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T08:22:51,617 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75fd415{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir/,AVAILABLE} 2024-11-17T08:22:51,618 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2544e7fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T08:22:51,719 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a6c60e4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/java.io.tmpdir/jetty-localhost-42283-hadoop-hdfs-3_4_1-tests_jar-_-any-1404925389612440376/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T08:22:51,720 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@41e0aaee{HTTP/1.1, (http/1.1)}{localhost:42283} 2024-11-17T08:22:51,720 INFO [Time-limited test {}] server.Server(415): Started @4784ms 2024-11-17T08:22:51,722 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T08:22:51,759 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T08:22:51,763 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T08:22:51,764 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T08:22:51,765 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T08:22:51,765 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T08:22:51,766 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c4dfe8c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir/,AVAILABLE} 2024-11-17T08:22:51,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b9fb405{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T08:22:51,866 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1738cb2d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/java.io.tmpdir/jetty-localhost-37931-hadoop-hdfs-3_4_1-tests_jar-_-any-14719940732195389840/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T08:22:51,866 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1473b53e{HTTP/1.1, (http/1.1)}{localhost:37931} 2024-11-17T08:22:51,867 INFO [Time-limited test {}] server.Server(415): Started @4931ms 2024-11-17T08:22:51,869 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T08:22:53,301 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data1/current/BP-2071561254-172.17.0.3-1731831769523/current, will proceed with Du for space computation calculation, 2024-11-17T08:22:53,301 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data2/current/BP-2071561254-172.17.0.3-1731831769523/current, will proceed with Du for space computation calculation, 2024-11-17T08:22:53,301 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data4/current/BP-2071561254-172.17.0.3-1731831769523/current, will proceed with Du for space computation calculation, 2024-11-17T08:22:53,301 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data3/current/BP-2071561254-172.17.0.3-1731831769523/current, will proceed with Du for space computation calculation, 2024-11-17T08:22:53,335 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data5/current/BP-2071561254-172.17.0.3-1731831769523/current, will proceed with Du for space computation calculation, 2024-11-17T08:22:53,337 WARN [Thread-134 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data6/current/BP-2071561254-172.17.0.3-1731831769523/current, will proceed with Du for space computation calculation, 2024-11-17T08:22:53,362 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T08:22:53,362 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T08:22:53,366 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T08:22:53,410 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c0a23165543f3b6 with lease ID 0x45f16b1255f20d95: Processing first storage report for DS-4dba2c6a-a6e9-493c-bf1f-0b11f461aff0 from datanode DatanodeRegistration(127.0.0.1:45541, datanodeUuid=4c783461-adb2-4e93-a110-b43275127ba1, infoPort=38395, infoSecurePort=0, ipcPort=37213, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523) 2024-11-17T08:22:53,411 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c0a23165543f3b6 with lease ID 0x45f16b1255f20d95: from storage DS-4dba2c6a-a6e9-493c-bf1f-0b11f461aff0 node DatanodeRegistration(127.0.0.1:45541, datanodeUuid=4c783461-adb2-4e93-a110-b43275127ba1, infoPort=38395, infoSecurePort=0, ipcPort=37213, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-17T08:22:53,412 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd3d659910640d58c with lease ID 0x45f16b1255f20d94: Processing first storage report for DS-b8c907ca-2e8c-4a7f-b36b-2c33a25a594d from datanode DatanodeRegistration(127.0.0.1:46265, datanodeUuid=1db8413a-aa16-40d9-a659-93add888ceb8, infoPort=38255, infoSecurePort=0, ipcPort=43827, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523) 2024-11-17T08:22:53,412 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd3d659910640d58c with lease ID 0x45f16b1255f20d94: from storage DS-b8c907ca-2e8c-4a7f-b36b-2c33a25a594d node DatanodeRegistration(127.0.0.1:46265, datanodeUuid=1db8413a-aa16-40d9-a659-93add888ceb8, infoPort=38255, infoSecurePort=0, ipcPort=43827, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T08:22:53,412 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4af39fe2909248ff with lease ID 0x45f16b1255f20d93: Processing first storage report for DS-5c4838e4-db9f-4274-9cbb-94d66f59e616 from datanode DatanodeRegistration(127.0.0.1:44055, datanodeUuid=f36dad06-3263-4aa1-89b9-ad3b577b967f, infoPort=45385, infoSecurePort=0, ipcPort=35931, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523) 2024-11-17T08:22:53,412 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4af39fe2909248ff with lease ID 0x45f16b1255f20d93: from storage DS-5c4838e4-db9f-4274-9cbb-94d66f59e616 node DatanodeRegistration(127.0.0.1:44055, datanodeUuid=f36dad06-3263-4aa1-89b9-ad3b577b967f, infoPort=45385, infoSecurePort=0, ipcPort=35931, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T08:22:53,412 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c0a23165543f3b6 with lease ID 0x45f16b1255f20d95: Processing first storage report for DS-0bb071b5-c336-443d-8e7f-c3ad22f0f23b from datanode DatanodeRegistration(127.0.0.1:45541, datanodeUuid=4c783461-adb2-4e93-a110-b43275127ba1, infoPort=38395, infoSecurePort=0, ipcPort=37213, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523) 2024-11-17T08:22:53,413 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c0a23165543f3b6 with lease ID 0x45f16b1255f20d95: from storage DS-0bb071b5-c336-443d-8e7f-c3ad22f0f23b node DatanodeRegistration(127.0.0.1:45541, datanodeUuid=4c783461-adb2-4e93-a110-b43275127ba1, infoPort=38395, infoSecurePort=0, ipcPort=37213, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T08:22:53,413 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd3d659910640d58c with lease ID 0x45f16b1255f20d94: Processing first storage report for DS-f40b7a55-a13e-4b93-9683-edab23725d35 from datanode DatanodeRegistration(127.0.0.1:46265, datanodeUuid=1db8413a-aa16-40d9-a659-93add888ceb8, infoPort=38255, infoSecurePort=0, ipcPort=43827, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523) 2024-11-17T08:22:53,413 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd3d659910640d58c with lease ID 0x45f16b1255f20d94: from storage DS-f40b7a55-a13e-4b93-9683-edab23725d35 node DatanodeRegistration(127.0.0.1:46265, datanodeUuid=1db8413a-aa16-40d9-a659-93add888ceb8, infoPort=38255, infoSecurePort=0, ipcPort=43827, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T08:22:53,413 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4af39fe2909248ff with lease ID 0x45f16b1255f20d93: Processing first storage report for DS-f994815e-a416-4491-b547-2807c7958fc4 from datanode DatanodeRegistration(127.0.0.1:44055, datanodeUuid=f36dad06-3263-4aa1-89b9-ad3b577b967f, infoPort=45385, infoSecurePort=0, ipcPort=35931, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523) 2024-11-17T08:22:53,413 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4af39fe2909248ff with lease ID 0x45f16b1255f20d93: from storage DS-f994815e-a416-4491-b547-2807c7958fc4 node DatanodeRegistration(127.0.0.1:44055, datanodeUuid=f36dad06-3263-4aa1-89b9-ad3b577b967f, infoPort=45385, infoSecurePort=0, ipcPort=35931, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T08:22:53,496 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a 2024-11-17T08:22:53,506 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=155, OpenFileDescriptor=393, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3830 2024-11-17T08:22:53,509 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T08:22:54,061 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T08:22:54,075 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:54,141 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:54,178 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831774172 2024-11-17T08:22:54,347 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831774172 2024-11-17T08:22:54,390 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:54,447 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831774172, startPosition=0, beingWritten=true 2024-11-17T08:22:54,579 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831774172 to pos 203, reset compression=false 2024-11-17T08:22:54,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:54,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:54,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:54,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:54,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:54,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741825_1001 (size=447) 2024-11-17T08:22:54,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741825_1001 (size=447) 2024-11-17T08:22:54,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741825_1001 (size=447) 2024-11-17T08:22:55,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:55,049 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831774172) 2024-11-17T08:22:55,056 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=166 (was 155) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@3c413271 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: LeaseRenewer:jenkins@localhost:33161 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=411 (was 393) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3710 (was 3830) 2024-11-17T08:22:55,064 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=166, OpenFileDescriptor=411, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3709 2024-11-17T08:22:55,065 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:55,071 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:55,074 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074 2024-11-17T08:22:55,086 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074 2024-11-17T08:22:55,088 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:55,092 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074, startPosition=0, beingWritten=true 2024-11-17T08:22:55,176 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:22:55,179 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074, startPosition=0, beingWritten=true 2024-11-17T08:22:55,192 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:55,217 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074, startPosition=0, beingWritten=true 2024-11-17T08:22:55,227 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:55,258 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074, startPosition=0, beingWritten=true 2024-11-17T08:22:55,274 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:55,315 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074, startPosition=0, beingWritten=true 2024-11-17T08:22:55,327 DEBUG [Thread-175 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:22:55,327 DEBUG [Thread-175 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074 to pos 460, reset compression=false 2024-11-17T08:22:55,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741826_1002 (size=468) 2024-11-17T08:22:55,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741826_1002 (size=468) 2024-11-17T08:22:55,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741826_1002 (size=468) 2024-11-17T08:22:55,352 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:55,352 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831775074) 2024-11-17T08:22:55,358 DEBUG [Thread-175 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074 to pos 460, reset compression=false 2024-11-17T08:22:55,359 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=173 (was 166) Potentially hanging thread: Thread-175 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=415 (was 411) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3695 (was 3709) 2024-11-17T08:22:55,368 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=173, OpenFileDescriptor=415, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3693 2024-11-17T08:22:55,369 INFO [Thread-175 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074 2024-11-17T08:22:55,370 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:55,374 DEBUG [Thread-175 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831775074 2024-11-17T08:22:55,375 DEBUG [Thread-175 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:55,375 DEBUG [Thread-175 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:55,376 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:55,379 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1731831775378 2024-11-17T08:22:55,391 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1731831775378 2024-11-17T08:22:55,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:55,512 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.false 2024-11-17T08:22:55,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741828_1004 (size=231) 2024-11-17T08:22:55,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741828_1004 (size=231) 2024-11-17T08:22:55,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741828_1004 (size=231) 2024-11-17T08:22:55,530 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-17T08:22:55,531 DEBUG [Thread-199 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/user/jenkins/log.1.false, startPosition=0, beingWritten=false 2024-11-17T08:22:55,540 DEBUG [Thread-199 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/user/jenkins/log.1.false 2024-11-17T08:22:55,540 DEBUG [Thread-199 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:22:55,541 DEBUG [Thread-199 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/user/jenkins/log.2.false, startPosition=0, beingWritten=false 2024-11-17T08:22:55,543 WARN [Thread-199 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:33161/user/jenkins/log.2.false, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-17T08:22:55,544 DEBUG [Thread-199 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/user/jenkins/log.2.false 2024-11-17T08:22:55,544 DEBUG [Thread-199 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:55,544 DEBUG [Thread-199 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:55,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741827_1003 (size=93) 2024-11-17T08:22:55,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741827_1003 (size=93) 2024-11-17T08:22:55,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741827_1003 (size=93) 2024-11-17T08:22:55,554 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:55,555 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831775378) 2024-11-17T08:22:55,561 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=178 (was 173) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 415) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3678 (was 3693) 2024-11-17T08:22:55,568 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=178, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3677 2024-11-17T08:22:55,569 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:55,574 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:55,577 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 2024-11-17T08:22:55,589 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 2024-11-17T08:22:55,590 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:55,591 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591 2024-11-17T08:22:55,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,603 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591 2024-11-17T08:22:55,608 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:55,608 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 is not closed yet, will try archiving it next time 2024-11-17T08:22:55,610 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:55,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741829_1005 (size=93) 2024-11-17T08:22:55,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741829_1005 (size=93) 2024-11-17T08:22:55,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741829_1005 (size=93) 2024-11-17T08:22:55,612 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 2024-11-17T08:22:55,713 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 2024-11-17T08:22:55,730 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:22:55,731 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:55,731 DEBUG [Thread-216 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577, startPosition=0, beingWritten=false 2024-11-17T08:22:55,734 INFO [Thread-216 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 2024-11-17T08:22:55,735 WARN [Thread-216 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-17T08:22:55,736 DEBUG [Thread-216 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775577 2024-11-17T08:22:55,736 DEBUG [Thread-216 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591, startPosition=0, beingWritten=true 2024-11-17T08:22:55,755 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591 to pos 0, reset compression=false 2024-11-17T08:22:55,776 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591 to pos 0, reset compression=false 2024-11-17T08:22:55,798 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591 to pos 0, reset compression=false 2024-11-17T08:22:55,816 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591 to pos 0, reset compression=false 2024-11-17T08:22:55,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,835 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591 to pos 0, reset compression=false 2024-11-17T08:22:55,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741830_1006 (size=93) 2024-11-17T08:22:55,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741830_1006 (size=93) 2024-11-17T08:22:55,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741830_1006 (size=93) 2024-11-17T08:22:55,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:55,844 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831775591) 2024-11-17T08:22:55,845 DEBUG [Thread-216 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731831775591 2024-11-17T08:22:55,846 DEBUG [Thread-216 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:55,846 DEBUG [Thread-216 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:55,851 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=184 (was 178) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-216 app//org.apache.logging.log4j.core.config.AppenderControl.isFilteredByAppender(AppenderControl.java:156) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:133) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2034) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1899) app//org.apache.logging.slf4j.Log4jLogger.debug(Log4jLogger.java:124) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.dequeueCurrentLog(WALEntryStream.java:456) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:374) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3672 (was 3677) 2024-11-17T08:22:55,859 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=184, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3672 2024-11-17T08:22:55,860 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:55,865 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:55,866 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775866 2024-11-17T08:22:55,876 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775866 2024-11-17T08:22:55,877 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:55,881 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775881 2024-11-17T08:22:55,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,893 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775866 with entries=10, filesize=1.30 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775881 2024-11-17T08:22:55,894 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:55,894 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775866 is not closed yet, will try archiving it next time 2024-11-17T08:22:55,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741831_1007 (size=1343) 2024-11-17T08:22:55,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741831_1007 (size=1343) 2024-11-17T08:22:55,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741831_1007 (size=1343) 2024-11-17T08:22:55,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:55,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741832_1008 (size=718) 2024-11-17T08:22:55,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741832_1008 (size=718) 2024-11-17T08:22:55,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741832_1008 (size=718) 2024-11-17T08:22:55,908 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-17T08:22:55,909 DEBUG [Thread-239 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775866, startPosition=0, beingWritten=false 2024-11-17T08:22:55,919 DEBUG [Thread-239 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-17T08:22:55,920 DEBUG [Thread-239 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775866 2024-11-17T08:22:55,920 DEBUG [Thread-239 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775881, startPosition=0, beingWritten=false 2024-11-17T08:22:55,929 DEBUG [Thread-239 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731831775881 2024-11-17T08:22:55,930 DEBUG [Thread-239 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-17T08:22:55,930 DEBUG [Thread-239 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:55,930 DEBUG [Thread-239 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:55,937 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:55,937 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831775881) 2024-11-17T08:22:55,945 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=189 (was 184) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3669 (was 3672) 2024-11-17T08:22:55,954 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=189, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3668 2024-11-17T08:22:55,955 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:55,961 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:55,962 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831775962 2024-11-17T08:22:55,976 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831775962 2024-11-17T08:22:55,977 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:55,993 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-17T08:22:55,995 DEBUG [Thread-249 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-17T08:22:55,997 WARN [Thread-249 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-17T08:22:55,998 DEBUG [Thread-249 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-17T08:22:55,999 DEBUG [Thread-249 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:55,999 DEBUG [Thread-249 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:56,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741833_1009 (size=93) 2024-11-17T08:22:56,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741833_1009 (size=93) 2024-11-17T08:22:56,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741833_1009 (size=93) 2024-11-17T08:22:56,012 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:56,013 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831775962) 2024-11-17T08:22:56,020 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=194 (was 189) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3666 (was 3668) 2024-11-17T08:22:56,028 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=194, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3666 2024-11-17T08:22:56,030 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:56,035 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:56,037 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776036 2024-11-17T08:22:56,052 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776036 2024-11-17T08:22:56,056 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:56,060 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776059 2024-11-17T08:22:56,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,074 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776036 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776059 2024-11-17T08:22:56,075 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:56,075 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776036 is not closed yet, will try archiving it next time 2024-11-17T08:22:56,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741834_1010 (size=218) 2024-11-17T08:22:56,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741834_1010 (size=218) 2024-11-17T08:22:56,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741834_1010 (size=218) 2024-11-17T08:22:56,086 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:56,089 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:22:56,090 DEBUG [Thread-266 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776036, startPosition=0, beingWritten=false 2024-11-17T08:22:56,096 DEBUG [Thread-266 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776036 2024-11-17T08:22:56,096 DEBUG [Thread-266 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-17T08:22:56,096 DEBUG [Thread-266 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776059, startPosition=0, beingWritten=true 2024-11-17T08:22:56,115 DEBUG [Thread-266 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-17T08:22:56,116 DEBUG [Thread-266 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776059 to pos 2585, reset compression=false 2024-11-17T08:22:56,116 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 2024-11-17T08:22:56,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,129 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776059 with entries=20, filesize=2.52 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 2024-11-17T08:22:56,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741835_1011 (size=2593) 2024-11-17T08:22:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741835_1011 (size=2593) 2024-11-17T08:22:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741835_1011 (size=2593) 2024-11-17T08:22:56,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:56,145 DEBUG [Thread-266 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776059 for the last time to pos 2585, reset compression=false 2024-11-17T08:22:56,150 DEBUG [Thread-266 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776059 and length of the file is 2593 2024-11-17T08:22:56,150 DEBUG [Thread-266 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776059 2024-11-17T08:22:56,151 DEBUG [Thread-266 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116, startPosition=0, beingWritten=true 2024-11-17T08:22:56,164 DEBUG [Thread-266 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-17T08:22:56,165 DEBUG [Thread-266 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 to pos 1335, reset compression=false 2024-11-17T08:22:56,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741836_1012 (size=1343) 2024-11-17T08:22:56,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741836_1012 (size=1343) 2024-11-17T08:22:56,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741836_1012 (size=1343) 2024-11-17T08:22:56,186 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:56,187 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831776116) 2024-11-17T08:22:56,196 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=200 (was 194) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-266 java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method) java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017) java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method) java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017) java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto.toBuilder(DescriptorProtos.java:2114) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto.newBuilder(DescriptorProtos.java:2107) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto$1.parsePartialFrom(DescriptorProtos.java:4485) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto$1.parsePartialFrom(DescriptorProtos.java:4479) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:135) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:168) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:180) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:185) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hbase.thirdparty.com.google.protobuf.DescriptorProtos$FileDescriptorProto.parseFrom(DescriptorProtos.java:2056) app//org.apache.hbase.thirdparty.com.google.protobuf.Descriptors$FileDescriptor.internalBuildGeneratedFileFrom(Descriptors.java:466) app//org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.(HBaseProtos.java:27113) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.(WALProtos.java:15202) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.internalGetFieldAccessorTable(WALProtos.java:1640) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.getDescriptorForType(GeneratedMessage.java:106) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:151) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:185) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.newUninitializedMessageException(AbstractMessage.java:446) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage.newUninitializedMessageException(AbstractMessage.java:284) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.newUninitializedMessageException(AbstractParser.java:30) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:44) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3646 (was 3666) 2024-11-17T08:22:56,208 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=200, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3645 2024-11-17T08:22:56,209 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:56,214 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALKeySerialization_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:56,215 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1731831776215 2024-11-17T08:22:56,231 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALKeySerialization_0_ isCompressionEnabled=false]/testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1731831776215 2024-11-17T08:22:56,237 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:56,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741837_1013 (size=93) 2024-11-17T08:22:56,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741837_1013 (size=93) 2024-11-17T08:22:56,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741837_1013 (size=93) 2024-11-17T08:22:56,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:56,258 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831776215) 2024-11-17T08:22:56,266 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=205 (was 200) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3639 (was 3645) 2024-11-17T08:22:56,273 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=205, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3638 2024-11-17T08:22:56,274 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:56,278 INFO [Thread-266 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=1335, currentPosition=1343 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-17T08:22:56,278 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:56,278 DEBUG [Thread-266 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 for the last time to pos 1335, reset compression=false 2024-11-17T08:22:56,279 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 2024-11-17T08:22:56,280 INFO [Thread-266 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 2024-11-17T08:22:56,287 INFO [Thread-266 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 2024-11-17T08:22:56,288 DEBUG [Thread-266 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 and length of the file is 1343 2024-11-17T08:22:56,288 DEBUG [Thread-266 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831776116 2024-11-17T08:22:56,289 DEBUG [Thread-266 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:56,289 DEBUG [Thread-266 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:56,299 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 2024-11-17T08:22:56,303 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:56,306 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279, startPosition=0, beingWritten=true 2024-11-17T08:22:56,316 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:22:56,319 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:56,420 DEBUG [Thread-296 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279, startPosition=0, beingWritten=true 2024-11-17T08:22:56,433 DEBUG [Thread-296 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:22:56,433 DEBUG [Thread-296 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 to pos 460, reset compression=false 2024-11-17T08:22:56,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741838_1014 (size=468) 2024-11-17T08:22:56,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741838_1014 (size=468) 2024-11-17T08:22:56,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741838_1014 (size=468) 2024-11-17T08:22:56,442 INFO [Thread-296 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=460, currentPosition=468 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-17T08:22:56,442 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:56,443 DEBUG [Thread-296 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 for the last time to pos 460, reset compression=false 2024-11-17T08:22:56,443 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831776279) 2024-11-17T08:22:56,446 INFO [Thread-296 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 2024-11-17T08:22:56,451 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=213 (was 205) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-296 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy46.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$351/0x00007f6a148a1db0.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) jdk.internal.reflect.GeneratedMethodAccessor32.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy47.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3625 (was 3638) 2024-11-17T08:22:56,453 INFO [Thread-296 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 2024-11-17T08:22:56,454 DEBUG [Thread-296 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 and length of the file is 468 2024-11-17T08:22:56,454 DEBUG [Thread-296 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731831776279 2024-11-17T08:22:56,454 DEBUG [Thread-296 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:56,454 DEBUG [Thread-296 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:56,458 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=213, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3625 2024-11-17T08:22:56,459 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:56,463 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:56,464 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776464 2024-11-17T08:22:56,472 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776464 2024-11-17T08:22:56,473 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:56,478 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:56,478 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776464, startPosition=0, beingWritten=true 2024-11-17T08:22:56,487 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776487 2024-11-17T08:22:56,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,500 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776464 with entries=3, filesize=439 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776487 2024-11-17T08:22:56,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741839_1015 (size=447) 2024-11-17T08:22:56,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741839_1015 (size=447) 2024-11-17T08:22:56,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741839_1015 (size=447) 2024-11-17T08:22:56,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:56,510 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:56,511 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:56,511 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776464 for the last time to pos 321, reset compression=false 2024-11-17T08:22:56,516 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:56,516 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776464 2024-11-17T08:22:56,617 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731831776487, startPosition=0, beingWritten=true 2024-11-17T08:22:56,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741840_1016 (size=211) 2024-11-17T08:22:56,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741840_1016 (size=211) 2024-11-17T08:22:56,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741840_1016 (size=211) 2024-11-17T08:22:56,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:56,637 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831776487) 2024-11-17T08:22:56,645 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=218 (was 213) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3616 (was 3625) 2024-11-17T08:22:56,653 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=218, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3616 2024-11-17T08:22:56,655 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:56,659 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:56,660 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776660 2024-11-17T08:22:56,670 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776660 2024-11-17T08:22:56,674 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:56,677 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:56,677 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776660, startPosition=0, beingWritten=true 2024-11-17T08:22:56,684 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776684 2024-11-17T08:22:56,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,693 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776660 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776684 2024-11-17T08:22:56,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:56,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776660 is not closed yet, will try archiving it next time 2024-11-17T08:22:56,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741841_1017 (size=218) 2024-11-17T08:22:56,697 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:56,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741841_1017 (size=218) 2024-11-17T08:22:56,697 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776660 for the last time to pos 210, reset compression=false 2024-11-17T08:22:56,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741841_1017 (size=218) 2024-11-17T08:22:56,703 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776660 and length of the file is 218 2024-11-17T08:22:56,704 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776660 2024-11-17T08:22:56,804 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731831776684, startPosition=0, beingWritten=true 2024-11-17T08:22:56,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:56,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741842_1018 (size=218) 2024-11-17T08:22:56,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741842_1018 (size=218) 2024-11-17T08:22:56,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741842_1018 (size=218) 2024-11-17T08:22:56,824 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:56,825 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831776684) 2024-11-17T08:22:56,834 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=223 (was 218) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3619 (was 3616) - AvailableMemoryMB LEAK? - 2024-11-17T08:22:56,842 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=223, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3619 2024-11-17T08:22:56,844 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:56,848 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:56,849 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731831776849 2024-11-17T08:22:56,862 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731831776849 2024-11-17T08:22:56,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:56,871 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731831776849, startPosition=0, beingWritten=true 2024-11-17T08:22:56,883 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-17T08:22:57,884 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731831776849 to pos 203, reset compression=false 2024-11-17T08:22:57,896 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-17T08:22:57,897 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731831776849 to pos 203, reset compression=false 2024-11-17T08:22:57,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741843_1019 (size=329) 2024-11-17T08:22:57,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741843_1019 (size=329) 2024-11-17T08:22:57,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741843_1019 (size=329) 2024-11-17T08:22:57,910 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:57,910 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831776849) 2024-11-17T08:22:57,919 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=228 (was 223) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3562 (was 3619) 2024-11-17T08:22:57,939 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=228, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3561 2024-11-17T08:22:57,942 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:57,946 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:57,946 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831777946 2024-11-17T08:22:57,954 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831777946 2024-11-17T08:22:57,956 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:57,960 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831777946, startPosition=0, beingWritten=true 2024-11-17T08:22:57,972 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831777946, startPosition=203, beingWritten=true 2024-11-17T08:22:57,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:57,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741844_1020 (size=447) 2024-11-17T08:22:57,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741844_1020 (size=447) 2024-11-17T08:22:57,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741844_1020 (size=447) 2024-11-17T08:22:57,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:57,990 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831777946) 2024-11-17T08:22:57,998 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=233 (was 228) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3554 (was 3561) 2024-11-17T08:22:58,006 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=233, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3554 2024-11-17T08:22:58,007 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:58,012 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:58,013 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731831778013 2024-11-17T08:22:58,022 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731831778013 2024-11-17T08:22:58,023 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:58,026 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731831778013, startPosition=0, beingWritten=true 2024-11-17T08:22:58,036 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:22:58,037 DEBUG [Thread-366 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731831778013, startPosition=0, beingWritten=true 2024-11-17T08:22:58,044 DEBUG [Thread-366 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:22:58,045 DEBUG [Thread-366 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731831778013 to pos 460, reset compression=false 2024-11-17T08:22:58,051 DEBUG [Thread-366 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-17T08:22:58,051 DEBUG [Thread-366 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731831778013 to pos 580, reset compression=false 2024-11-17T08:22:58,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741845_1021 (size=588) 2024-11-17T08:22:58,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741845_1021 (size=588) 2024-11-17T08:22:58,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741845_1021 (size=588) 2024-11-17T08:22:58,058 DEBUG [Thread-366 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731831778013 2024-11-17T08:22:58,059 DEBUG [Thread-366 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:58,059 DEBUG [Thread-366 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:58,059 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:58,059 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831778013) 2024-11-17T08:22:58,067 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=239 (was 233) Potentially hanging thread: Thread-366 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=428 (was 427) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3546 (was 3554) 2024-11-17T08:22:58,074 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=239, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3545 2024-11-17T08:22:58,075 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:58,079 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:58,080 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 2024-11-17T08:22:58,091 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 2024-11-17T08:22:58,092 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:58,093 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778093 2024-11-17T08:22:58,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,104 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778093 2024-11-17T08:22:58,105 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:58,105 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 is not closed yet, will try archiving it next time 2024-11-17T08:22:58,106 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:58,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741846_1022 (size=93) 2024-11-17T08:22:58,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741846_1022 (size=93) 2024-11-17T08:22:58,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741846_1022 (size=93) 2024-11-17T08:22:58,109 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 2024-11-17T08:22:58,210 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 2024-11-17T08:22:58,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741848_1024 (size=85) 2024-11-17T08:22:58,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741848_1024 (size=85) 2024-11-17T08:22:58,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741848_1024 (size=85) 2024-11-17T08:22:58,235 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080, startPosition=0, beingWritten=false 2024-11-17T08:22:58,237 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 2024-11-17T08:22:58,242 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 for the last time to pos 0, reset compression=false 2024-11-17T08:22:58,244 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 2024-11-17T08:22:58,248 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 2024-11-17T08:22:58,249 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 and length of the file is 85 2024-11-17T08:22:58,249 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778080 2024-11-17T08:22:58,249 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731831778093, startPosition=0, beingWritten=true 2024-11-17T08:22:58,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741847_1023 (size=218) 2024-11-17T08:22:58,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741847_1023 (size=218) 2024-11-17T08:22:58,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741847_1023 (size=218) 2024-11-17T08:22:58,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:58,262 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831778093) 2024-11-17T08:22:58,270 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=244 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3536 (was 3545) 2024-11-17T08:22:58,279 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=244, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=3536 2024-11-17T08:22:58,280 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:58,284 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:58,285 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 2024-11-17T08:22:58,295 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 2024-11-17T08:22:58,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:58,302 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, startPosition=0, beingWritten=true 2024-11-17T08:22:58,313 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:22:58,314 DEBUG [Thread-404 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, startPosition=0, beingWritten=true 2024-11-17T08:22:58,322 WARN [Thread-404 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:58,343 DEBUG [Thread-404 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, startPosition=0, beingWritten=true 2024-11-17T08:22:58,350 WARN [Thread-404 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:58,381 DEBUG [Thread-404 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, startPosition=0, beingWritten=true 2024-11-17T08:22:58,390 WARN [Thread-404 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:58,431 DEBUG [Thread-404 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, startPosition=0, beingWritten=true 2024-11-17T08:22:58,446 WARN [Thread-404 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:58,497 DEBUG [Thread-404 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, startPosition=0, beingWritten=true 2024-11-17T08:22:58,504 WARN [Thread-404 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:58,565 DEBUG [Thread-404 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, startPosition=0, beingWritten=true 2024-11-17T08:22:58,596 DEBUG [Thread-404 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:22:58,597 DEBUG [Thread-404 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 to pos 460, reset compression=false 2024-11-17T08:22:58,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741849_1025 (size=468) 2024-11-17T08:22:58,614 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Sending block BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:44055, datanodeUuid=f36dad06-3263-4aa1-89b9-ad3b577b967f, infoPort=45385, infoSecurePort=0, ipcPort=35931, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523):Got exception while serving BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025 to /127.0.0.1:48524 java.io.FileNotFoundException: BlockId 1073741849 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:22:58,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741849_1025 (size=468) 2024-11-17T08:22:58,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741849_1025 (size=468) 2024-11-17T08:22:58,615 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48524 [Sending block BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025] {}] datanode.DataXceiver(331): 127.0.0.1:44055:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:48524 dst: /127.0.0.1:44055 java.io.FileNotFoundException: BlockId 1073741849 is not valid. at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:810) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getBlockReplica(FsDatasetImpl.java:801) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getMetaDataInputStream(FsDatasetImpl.java:246) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:319) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:22:58,623 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:58,623 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831778285) 2024-11-17T08:22:58,630 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48790 [Sending block BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:44055, datanodeUuid=f36dad06-3263-4aa1-89b9-ad3b577b967f, infoPort=45385, infoSecurePort=0, ipcPort=35931, storageInfo=lv=-57;cid=testClusterID;nsid=2124270823;c=1731831769523):Got exception while serving BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025 to /127.0.0.1:48790 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:22:58,630 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48790 [Sending block BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025] {}] datanode.DataXceiver(331): 127.0.0.1:44055:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:48790 dst: /127.0.0.1:44055 org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue. at org.apache.hadoop.hdfs.server.datanode.BlockSender.getReplica(BlockSender.java:509) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:261) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:22:58,630 WARN [Thread-404 {}] impl.BlockReaderFactory(772): I/O error constructing remote block reader for block BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025 java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:48790, remote=/127.0.0.1:44055, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, for pool BP-2071561254-172.17.0.3-1731831769523 block 1073741849_1025 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-17T08:22:58,631 WARN [Thread-404 {}] hdfs.DFSInputStream(667): Failed to connect to /127.0.0.1:44055 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 for block BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025, add to deadNodes and continue. java.io.IOException: Got error, status=ERROR, status message opReadBlock BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-2071561254-172.17.0.3-1731831769523:blk_1073741849_1025. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue., for OP_READ_BLOCK, self=/127.0.0.1:48790, remote=/127.0.0.1:44055, for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285, for pool BP-2071561254-172.17.0.3-1731831769523 block 1073741849_1025 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.checkSuccess(BlockReaderRemote.java:452) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:420) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-17T08:22:58,634 INFO [Thread-404 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=460, currentPosition=468 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-17T08:22:58,634 DEBUG [Thread-404 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 for the last time to pos 460, reset compression=false 2024-11-17T08:22:58,638 INFO [Thread-404 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 2024-11-17T08:22:58,638 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=249 (was 244) Potentially hanging thread: Thread-404 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:419) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) app//org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) app//org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.DataInputStream.readInt(DataInputStream.java:381) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=426 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=3452 (was 3536) 2024-11-17T08:22:58,646 INFO [Thread-404 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 2024-11-17T08:22:58,647 DEBUG [Thread-404 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 and length of the file is 468 2024-11-17T08:22:58,647 DEBUG [Thread-404 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731831778285 2024-11-17T08:22:58,648 DEBUG [Thread-404 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:58,648 DEBUG [Thread-404 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:58,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=249, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3450 2024-11-17T08:22:58,650 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:58,655 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEmptyStream_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:58,656 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731831778656 2024-11-17T08:22:58,666 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731831778656 2024-11-17T08:22:58,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:58,669 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731831778656, startPosition=0, beingWritten=true 2024-11-17T08:22:58,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741850_1026 (size=93) 2024-11-17T08:22:58,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741850_1026 (size=93) 2024-11-17T08:22:58,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741850_1026 (size=93) 2024-11-17T08:22:58,685 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:58,685 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831778656) 2024-11-17T08:22:58,695 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=254 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3446 (was 3450) 2024-11-17T08:22:58,704 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=254, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3445 2024-11-17T08:22:58,709 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:58,713 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:58,714 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778714 2024-11-17T08:22:58,734 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778714 2024-11-17T08:22:58,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:58,747 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778714, startPosition=0, beingWritten=true 2024-11-17T08:22:58,761 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:58,761 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778714, startPosition=210, beingWritten=true 2024-11-17T08:22:58,772 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778772 2024-11-17T08:22:58,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,791 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778714 with entries=3, filesize=460 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778772 2024-11-17T08:22:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741851_1027 (size=468) 2024-11-17T08:22:58,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741851_1027 (size=468) 2024-11-17T08:22:58,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741851_1027 (size=468) 2024-11-17T08:22:58,800 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:58,803 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:58,803 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778714, startPosition=335, beingWritten=false 2024-11-17T08:22:58,807 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:58,807 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778714 2024-11-17T08:22:58,908 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731831778772, startPosition=0, beingWritten=true 2024-11-17T08:22:58,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741852_1028 (size=218) 2024-11-17T08:22:58,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741852_1028 (size=218) 2024-11-17T08:22:58,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741852_1028 (size=218) 2024-11-17T08:22:58,926 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:58,926 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831778772) 2024-11-17T08:22:58,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=260 (was 254) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3410 (was 3445) 2024-11-17T08:22:58,941 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=260, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3409 2024-11-17T08:22:58,942 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:58,945 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:58,946 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831778946 2024-11-17T08:22:58,953 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831778946 2024-11-17T08:22:58,954 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:58,958 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831778946, startPosition=0, beingWritten=true 2024-11-17T08:22:58,966 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731831778946, startPosition=210, beingWritten=true 2024-11-17T08:22:58,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:58,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741853_1029 (size=468) 2024-11-17T08:22:58,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741853_1029 (size=468) 2024-11-17T08:22:58,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741853_1029 (size=468) 2024-11-17T08:22:58,980 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:58,980 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831778946) 2024-11-17T08:22:58,989 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=265 (was 260) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3406 (was 3409) 2024-11-17T08:22:58,999 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=265, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3405 2024-11-17T08:22:59,001 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:59,004 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:59,005 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831779004 2024-11-17T08:22:59,012 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831779004 2024-11-17T08:22:59,013 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:59,015 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831779015 2024-11-17T08:22:59,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,025 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831779004 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831779015 2024-11-17T08:22:59,026 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:59,026 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831779004 is not closed yet, will try archiving it next time 2024-11-17T08:22:59,027 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:59,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741854_1030 (size=218) 2024-11-17T08:22:59,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741854_1030 (size=218) 2024-11-17T08:22:59,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741854_1030 (size=218) 2024-11-17T08:22:59,031 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831779004, startPosition=0, beingWritten=false 2024-11-17T08:22:59,035 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731831779004 2024-11-17T08:22:59,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741855_1031 (size=93) 2024-11-17T08:22:59,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741855_1031 (size=93) 2024-11-17T08:22:59,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741855_1031 (size=93) 2024-11-17T08:22:59,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:59,449 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1731831779015) 2024-11-17T08:22:59,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=270 (was 265) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3369 (was 3405) 2024-11-17T08:22:59,467 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=270, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3369 2024-11-17T08:22:59,468 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:59,473 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:59,474 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831779474 2024-11-17T08:22:59,485 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831779474 2024-11-17T08:22:59,489 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:59,492 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831779474, startPosition=0, beingWritten=true 2024-11-17T08:22:59,497 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831779474: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:22:59,503 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831779474 to pos 197, reset compression=false 2024-11-17T08:22:59,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741856_1032 (size=317) 2024-11-17T08:22:59,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741856_1032 (size=317) 2024-11-17T08:22:59,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741856_1032 (size=317) 2024-11-17T08:22:59,519 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:59,519 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831779474) 2024-11-17T08:22:59,530 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=275 (was 270) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3360 (was 3369) 2024-11-17T08:22:59,539 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=275, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3359 2024-11-17T08:22:59,541 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:59,545 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:59,546 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546 2024-11-17T08:22:59,559 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546 2024-11-17T08:22:59,563 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:59,569 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546, startPosition=0, beingWritten=true 2024-11-17T08:22:59,573 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T08:22:59,574 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:22:59,579 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:22:59,580 DEBUG [Thread-489 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546, startPosition=0, beingWritten=true 2024-11-17T08:22:59,588 DEBUG [Thread-489 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:22:59,591 WARN [Thread-489 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:59,611 DEBUG [Thread-489 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546, startPosition=0, beingWritten=true 2024-11-17T08:22:59,617 DEBUG [Thread-489 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:22:59,619 WARN [Thread-489 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:59,650 DEBUG [Thread-489 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546, startPosition=0, beingWritten=true 2024-11-17T08:22:59,656 DEBUG [Thread-489 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:22:59,659 WARN [Thread-489 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-17T08:22:59,699 DEBUG [Thread-489 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546, startPosition=0, beingWritten=true 2024-11-17T08:22:59,705 DEBUG [Thread-489 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:22:59,707 DEBUG [Thread-489 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:22:59,707 DEBUG [Thread-489 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546 to pos 322, reset compression=false 2024-11-17T08:22:59,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741857_1033 (size=330) 2024-11-17T08:22:59,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741857_1033 (size=330) 2024-11-17T08:22:59,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741857_1033 (size=330) 2024-11-17T08:22:59,725 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:59,725 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831779546) 2024-11-17T08:22:59,733 DEBUG [Thread-489 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546 to pos 322, reset compression=false 2024-11-17T08:22:59,734 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=282 (was 275) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-489 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3338 (was 3359) 2024-11-17T08:22:59,737 INFO [Thread-489 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546 2024-11-17T08:22:59,741 DEBUG [Thread-489 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831779546 2024-11-17T08:22:59,741 DEBUG [Thread-489 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:59,741 DEBUG [Thread-489 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:59,743 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=282, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3337 2024-11-17T08:22:59,743 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:59,746 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:59,747 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1731831779747 2024-11-17T08:22:59,755 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1731831779747 2024-11-17T08:22:59,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:22:59,765 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.true 2024-11-17T08:22:59,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741859_1035 (size=225) 2024-11-17T08:22:59,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741859_1035 (size=225) 2024-11-17T08:22:59,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741859_1035 (size=225) 2024-11-17T08:22:59,777 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-17T08:22:59,781 DEBUG [Thread-512 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/user/jenkins/log.1.true, startPosition=0, beingWritten=false 2024-11-17T08:22:59,783 DEBUG [Thread-512 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/user/jenkins/log.1.true: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:22:59,786 DEBUG [Thread-512 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/user/jenkins/log.1.true 2024-11-17T08:22:59,786 DEBUG [Thread-512 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:22:59,786 DEBUG [Thread-512 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/user/jenkins/log.2.true, startPosition=0, beingWritten=false 2024-11-17T08:22:59,787 WARN [Thread-512 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:33161/user/jenkins/log.2.true, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-17T08:22:59,788 DEBUG [Thread-512 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/user/jenkins/log.2.true 2024-11-17T08:22:59,788 DEBUG [Thread-512 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:22:59,788 DEBUG [Thread-512 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:22:59,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741858_1034 (size=93) 2024-11-17T08:22:59,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741858_1034 (size=93) 2024-11-17T08:22:59,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741858_1034 (size=93) 2024-11-17T08:22:59,795 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:22:59,796 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831779747) 2024-11-17T08:22:59,804 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=287 (was 282) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3329 (was 3337) 2024-11-17T08:22:59,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=287, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3325 2024-11-17T08:22:59,814 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:22:59,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:22:59,818 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 2024-11-17T08:22:59,825 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 2024-11-17T08:22:59,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:22:59,829 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 2024-11-17T08:22:59,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:22:59,838 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 2024-11-17T08:22:59,840 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:22:59,840 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 is not closed yet, will try archiving it next time 2024-11-17T08:22:59,840 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:59,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741860_1036 (size=93) 2024-11-17T08:22:59,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741860_1036 (size=93) 2024-11-17T08:22:59,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741860_1036 (size=93) 2024-11-17T08:22:59,842 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 2024-11-17T08:22:59,943 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 2024-11-17T08:22:59,952 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:22:59,952 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:22:59,953 DEBUG [Thread-529 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818, startPosition=0, beingWritten=false 2024-11-17T08:22:59,954 INFO [Thread-529 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 2024-11-17T08:22:59,956 WARN [Thread-529 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-17T08:22:59,956 DEBUG [Thread-529 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779818 2024-11-17T08:22:59,956 DEBUG [Thread-529 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829, startPosition=0, beingWritten=true 2024-11-17T08:22:59,961 DEBUG [Thread-529 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:22:59,972 DEBUG [Thread-529 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 to pos 0, reset compression=false 2024-11-17T08:22:59,990 DEBUG [Thread-529 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 to pos 0, reset compression=false 2024-11-17T08:23:00,006 DEBUG [Thread-529 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 to pos 0, reset compression=false 2024-11-17T08:23:00,021 DEBUG [Thread-529 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 to pos 0, reset compression=false 2024-11-17T08:23:00,039 DEBUG [Thread-529 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 to pos 0, reset compression=false 2024-11-17T08:23:00,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,057 DEBUG [Thread-529 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 to pos 0, reset compression=false 2024-11-17T08:23:00,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741861_1037 (size=93) 2024-11-17T08:23:00,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741861_1037 (size=93) 2024-11-17T08:23:00,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741861_1037 (size=93) 2024-11-17T08:23:00,086 DEBUG [Thread-529 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 for the last time to pos 0, reset compression=false 2024-11-17T08:23:00,089 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:00,089 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831779829) 2024-11-17T08:23:00,094 INFO [Thread-529 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 2024-11-17T08:23:00,095 DEBUG [Thread-529 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 and length of the file is 93 2024-11-17T08:23:00,095 DEBUG [Thread-529 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731831779829 2024-11-17T08:23:00,095 DEBUG [Thread-529 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:23:00,095 DEBUG [Thread-529 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:23:00,098 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=293 (was 287) Potentially hanging thread: Thread-529 java.base@17.0.11/java.net.URI$Parser.scan(URI.java:3124) java.base@17.0.11/java.net.URI$Parser.checkChars(URI.java:3145) java.base@17.0.11/java.net.URI$Parser.parseServer(URI.java:3383) java.base@17.0.11/java.net.URI$Parser.parseAuthority(URI.java:3284) java.base@17.0.11/java.net.URI$Parser.parseHierarchical(URI.java:3221) java.base@17.0.11/java.net.URI$Parser.parse(URI.java:3177) java.base@17.0.11/java.net.URI.(URI.java:623) java.base@17.0.11/java.net.URI.create(URI.java:904) app//org.apache.hadoop.net.NetUtils.createURI(NetUtils.java:284) app//org.apache.hadoop.net.NetUtils.createSocketAddr(NetUtils.java:242) app//org.apache.hadoop.net.NetUtils.createSocketAddr(NetUtils.java:226) app//org.apache.hadoop.hdfs.DFSInputStream.getBestNodeDNAddrPair(DFSInputStream.java:1075) app//org.apache.hadoop.hdfs.DFSInputStream.chooseDataNode(DFSInputStream.java:949) app//org.apache.hadoop.hdfs.DFSInputStream.chooseDataNode(DFSInputStream.java:931) app//org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:637) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hbase.io.util.StreamUtils.readRawVarint32(StreamUtils.java:54) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.skipHeader(ProtobufWALTailingReader.java:265) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:286) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:212) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #19] app//org.apache.hadoop.hdfs.server.datanode.BlockSender.close(BlockSender.java:482) app//org.apache.hadoop.io.IOUtils.cleanupWithLogger(IOUtils.java:260) app//org.apache.hadoop.io.IOUtils.closeStream(IOUtils.java:278) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:670) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #48] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #51] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3314 (was 3325) 2024-11-17T08:23:00,107 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=293, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3314 2024-11-17T08:23:00,108 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:00,112 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:00,113 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780113 2024-11-17T08:23:00,122 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780113 2024-11-17T08:23:00,127 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:00,130 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780130 2024-11-17T08:23:00,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,139 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780113 with entries=10, filesize=707 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780130 2024-11-17T08:23:00,141 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:00,141 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780113 is not closed yet, will try archiving it next time 2024-11-17T08:23:00,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741862_1038 (size=715) 2024-11-17T08:23:00,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741862_1038 (size=715) 2024-11-17T08:23:00,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741862_1038 (size=715) 2024-11-17T08:23:00,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741863_1039 (size=432) 2024-11-17T08:23:00,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741863_1039 (size=432) 2024-11-17T08:23:00,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741863_1039 (size=432) 2024-11-17T08:23:00,552 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-17T08:23:00,554 DEBUG [Thread-553 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780113, startPosition=0, beingWritten=false 2024-11-17T08:23:00,558 DEBUG [Thread-553 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780113: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:00,562 DEBUG [Thread-553 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-17T08:23:00,563 DEBUG [Thread-553 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780113 2024-11-17T08:23:00,563 DEBUG [Thread-553 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780130, startPosition=0, beingWritten=false 2024-11-17T08:23:00,566 DEBUG [Thread-553 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780130: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:00,570 DEBUG [Thread-553 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731831780130 2024-11-17T08:23:00,570 DEBUG [Thread-553 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-17T08:23:00,571 DEBUG [Thread-553 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:23:00,571 DEBUG [Thread-553 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:23:00,574 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:00,574 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831780130) 2024-11-17T08:23:00,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=298 (was 293) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #53] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3306 (was 3314) 2024-11-17T08:23:00,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=298, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3305 2024-11-17T08:23:00,591 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:00,594 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:00,594 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831780594 2024-11-17T08:23:00,602 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831780594 2024-11-17T08:23:00,604 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:23:00,612 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-17T08:23:00,612 DEBUG [Thread-563 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-17T08:23:00,614 WARN [Thread-563 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-17T08:23:00,614 DEBUG [Thread-563 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-17T08:23:00,614 DEBUG [Thread-563 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:23:00,614 DEBUG [Thread-563 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:23:00,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741864_1040 (size=93) 2024-11-17T08:23:00,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741864_1040 (size=93) 2024-11-17T08:23:00,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741864_1040 (size=93) 2024-11-17T08:23:00,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:00,620 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831780594) 2024-11-17T08:23:00,629 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=303 (was 298) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3301 (was 3305) 2024-11-17T08:23:00,638 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=303, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3301 2024-11-17T08:23:00,639 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:00,641 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:00,642 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780642 2024-11-17T08:23:00,649 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780642 2024-11-17T08:23:00,650 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:23:00,652 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651 2024-11-17T08:23:00,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,660 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780642 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651 2024-11-17T08:23:00,661 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:00,661 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780642 is not closed yet, will try archiving it next time 2024-11-17T08:23:00,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741865_1041 (size=212) 2024-11-17T08:23:00,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741865_1041 (size=212) 2024-11-17T08:23:00,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741865_1041 (size=212) 2024-11-17T08:23:00,666 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:00,668 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:23:00,669 DEBUG [Thread-580 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780642, startPosition=0, beingWritten=false 2024-11-17T08:23:00,672 DEBUG [Thread-580 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780642: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:00,673 DEBUG [Thread-580 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780642 2024-11-17T08:23:00,673 DEBUG [Thread-580 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-17T08:23:00,674 DEBUG [Thread-580 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651, startPosition=0, beingWritten=true 2024-11-17T08:23:00,679 DEBUG [Thread-580 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:00,686 DEBUG [Thread-580 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-17T08:23:00,686 DEBUG [Thread-580 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651 to pos 1281, reset compression=false 2024-11-17T08:23:00,687 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686 2024-11-17T08:23:00,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,695 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651 with entries=20, filesize=1.25 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686 2024-11-17T08:23:00,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:23:00,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651 is not closed yet, will try archiving it next time 2024-11-17T08:23:00,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741866_1042 (size=1289) 2024-11-17T08:23:00,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741866_1042 (size=1289) 2024-11-17T08:23:00,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741866_1042 (size=1289) 2024-11-17T08:23:00,717 DEBUG [Thread-580 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651 to pos 1281, reset compression=false 2024-11-17T08:23:00,719 DEBUG [Thread-580 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780651 2024-11-17T08:23:00,719 DEBUG [Thread-580 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686, startPosition=0, beingWritten=true 2024-11-17T08:23:00,725 DEBUG [Thread-580 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:00,730 DEBUG [Thread-580 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-17T08:23:00,730 DEBUG [Thread-580 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686 to pos 715, reset compression=false 2024-11-17T08:23:00,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741867_1043 (size=723) 2024-11-17T08:23:00,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741867_1043 (size=723) 2024-11-17T08:23:00,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741867_1043 (size=723) 2024-11-17T08:23:00,737 INFO [Thread-580 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=715, currentPosition=723 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-17T08:23:00,737 DEBUG [Thread-580 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686 for the last time to pos 715, reset compression=false 2024-11-17T08:23:00,739 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:00,739 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831780686) 2024-11-17T08:23:00,743 INFO [Thread-580 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686 2024-11-17T08:23:00,743 DEBUG [Thread-580 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686 and length of the file is 723 2024-11-17T08:23:00,744 DEBUG [Thread-580 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831780686 2024-11-17T08:23:00,744 DEBUG [Thread-580 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:23:00,744 DEBUG [Thread-580 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:23:00,748 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=309 (was 303) Potentially hanging thread: Thread-580 app//org.apache.hadoop.thirdparty.protobuf.CodedInputStream$ArrayDecoder.(CodedInputStream.java:588) app//org.apache.hadoop.thirdparty.protobuf.CodedInputStream$ArrayDecoder.(CodedInputStream.java:575) app//org.apache.hadoop.thirdparty.protobuf.CodedInputStream.newInstance(CodedInputStream.java:123) app//org.apache.hadoop.thirdparty.protobuf.CodedInputStream.newInstance(CodedInputStream.java:117) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:134) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parseFrom(AbstractParser.java:168) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parseFrom(AbstractParser.java:180) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parseFrom(AbstractParser.java:185) app//org.apache.hadoop.thirdparty.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.parseFrom(DataTransferProtos.java:23686) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader.setFieldsFromData(PacketHeader.java:130) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:189) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:112) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.readNextPacket(BlockReaderRemote.java:187) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.read(BlockReaderRemote.java:146) app//org.apache.hadoop.hdfs.ByteArrayStrategy.readFromBlock(ReaderStrategy.java:118) app//org.apache.hadoop.hdfs.DFSInputStream.readBuffer(DFSInputStream.java:789) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:855) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.DataInputStream.readInt(DataInputStream.java:381) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #51] app//org.apache.hadoop.thirdparty.protobuf.CodedOutputStream$OutputStreamEncoder.writeFixed32(CodedOutputStream.java:2696) app//org.apache.hadoop.thirdparty.protobuf.CodedOutputStream.writeSFixed32(CodedOutputStream.java:228) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.writeTo(DataTransferProtos.java:23548) app//org.apache.hadoop.thirdparty.protobuf.AbstractMessageLite.writeTo(AbstractMessageLite.java:60) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader.putInBuffer(PacketHeader.java:167) app//org.apache.hadoop.hdfs.server.datanode.BlockSender.writePacketHeader(BlockSender.java:910) app//org.apache.hadoop.hdfs.server.datanode.BlockSender.sendPacket(BlockSender.java:571) app//org.apache.hadoop.hdfs.server.datanode.BlockSender.doSendBlock(BlockSender.java:829) app//org.apache.hadoop.hdfs.server.datanode.BlockSender.sendBlock(BlockSender.java:766) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:621) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #56] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3288 (was 3301) 2024-11-17T08:23:00,756 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=309, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3287 2024-11-17T08:23:00,757 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:00,760 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALKeySerialization_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:00,761 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1731831780761 2024-11-17T08:23:00,770 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALKeySerialization_1_ isCompressionEnabled=true]/testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1731831780761 2024-11-17T08:23:00,774 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:00,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741868_1044 (size=93) 2024-11-17T08:23:00,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741868_1044 (size=93) 2024-11-17T08:23:00,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741868_1044 (size=93) 2024-11-17T08:23:00,782 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:00,782 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831780761) 2024-11-17T08:23:00,791 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=314 (was 309) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3285 (was 3287) 2024-11-17T08:23:00,800 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=314, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3285 2024-11-17T08:23:00,800 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:00,803 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:00,804 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 2024-11-17T08:23:00,811 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 2024-11-17T08:23:00,812 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:23:00,816 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804, startPosition=0, beingWritten=true 2024-11-17T08:23:00,820 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:00,822 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:23:00,823 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:00,925 DEBUG [Thread-610 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804, startPosition=0, beingWritten=true 2024-11-17T08:23:00,932 DEBUG [Thread-610 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:00,935 DEBUG [Thread-610 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:23:00,936 DEBUG [Thread-610 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 to pos 314, reset compression=false 2024-11-17T08:23:00,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741869_1045 (size=322) 2024-11-17T08:23:00,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741869_1045 (size=322) 2024-11-17T08:23:00,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741869_1045 (size=322) 2024-11-17T08:23:00,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:00,944 INFO [Thread-610 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=314, currentPosition=322 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-17T08:23:00,944 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831780804) 2024-11-17T08:23:00,944 DEBUG [Thread-610 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 for the last time to pos 314, reset compression=false 2024-11-17T08:23:00,948 INFO [Thread-610 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 2024-11-17T08:23:00,953 INFO [Thread-610 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 2024-11-17T08:23:00,954 DEBUG [Thread-610 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 and length of the file is 322 2024-11-17T08:23:00,954 DEBUG [Thread-610 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731831780804 2024-11-17T08:23:00,954 DEBUG [Thread-610 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:23:00,954 DEBUG [Thread-610 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:23:00,955 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=321 (was 314) Potentially hanging thread: Thread-610 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy46.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$351/0x00007f6a148a1db0.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) jdk.internal.reflect.GeneratedMethodAccessor32.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy47.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #61] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3279 (was 3285) 2024-11-17T08:23:00,964 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=321, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3278 2024-11-17T08:23:00,965 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:00,969 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:00,970 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780970 2024-11-17T08:23:00,977 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780970 2024-11-17T08:23:00,978 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:00,981 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:00,981 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780970, startPosition=0, beingWritten=true 2024-11-17T08:23:00,985 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780970: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:00,988 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780988 2024-11-17T08:23:00,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:00,998 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780970 with entries=3, filesize=309 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780988 2024-11-17T08:23:01,002 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:01,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741870_1046 (size=317) 2024-11-17T08:23:01,002 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780970 is not closed yet, will try archiving it next time 2024-11-17T08:23:01,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741870_1046 (size=317) 2024-11-17T08:23:01,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741870_1046 (size=317) 2024-11-17T08:23:01,004 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:01,004 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:01,004 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780970 for the last time to pos 253, reset compression=false 2024-11-17T08:23:01,007 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:01,007 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780970 2024-11-17T08:23:01,108 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780988, startPosition=0, beingWritten=true 2024-11-17T08:23:01,113 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731831780988: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:01,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741871_1047 (size=205) 2024-11-17T08:23:01,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741871_1047 (size=205) 2024-11-17T08:23:01,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741871_1047 (size=205) 2024-11-17T08:23:01,123 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:01,123 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831780988) 2024-11-17T08:23:01,132 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=326 (was 321) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3267 (was 3278) 2024-11-17T08:23:01,141 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=326, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3267 2024-11-17T08:23:01,142 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:01,146 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:01,147 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146 2024-11-17T08:23:01,162 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146 2024-11-17T08:23:01,166 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:23:01,170 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:01,170 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146, startPosition=0, beingWritten=true 2024-11-17T08:23:01,176 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:01,177 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781177 2024-11-17T08:23:01,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,185 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781177 2024-11-17T08:23:01,189 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:23:01,189 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146 is not closed yet, will try archiving it next time 2024-11-17T08:23:01,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741872_1048 (size=212) 2024-11-17T08:23:01,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741872_1048 (size=212) 2024-11-17T08:23:01,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741872_1048 (size=212) 2024-11-17T08:23:01,194 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:01,195 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146 for the last time to pos 204, reset compression=false 2024-11-17T08:23:01,198 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146 and length of the file is 212 2024-11-17T08:23:01,198 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781146 2024-11-17T08:23:01,299 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781177, startPosition=0, beingWritten=true 2024-11-17T08:23:01,309 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731831781177: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:01,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:01,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741873_1049 (size=212) 2024-11-17T08:23:01,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741873_1049 (size=212) 2024-11-17T08:23:01,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741873_1049 (size=212) 2024-11-17T08:23:01,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:01,317 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831781177) 2024-11-17T08:23:01,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=331 (was 326) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #60] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #64] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3256 (was 3267) 2024-11-17T08:23:01,335 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=331, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3255 2024-11-17T08:23:01,336 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:01,339 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:01,340 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731831781340 2024-11-17T08:23:01,349 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731831781340 2024-11-17T08:23:01,352 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:01,359 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731831781340, startPosition=0, beingWritten=true 2024-11-17T08:23:01,363 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731831781340: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:01,364 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-17T08:23:02,365 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731831781340 to pos 197, reset compression=true 2024-11-17T08:23:02,372 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-17T08:23:02,373 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731831781340 to pos 197, reset compression=true 2024-11-17T08:23:02,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741874_1050 (size=261) 2024-11-17T08:23:02,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741874_1050 (size=261) 2024-11-17T08:23:02,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741874_1050 (size=261) 2024-11-17T08:23:02,387 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:02,387 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831781340) 2024-11-17T08:23:02,395 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=336 (was 331) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #64] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3263 (was 3255) - AvailableMemoryMB LEAK? - 2024-11-17T08:23:02,405 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=339, OpenFileDescriptor=428, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3263 2024-11-17T08:23:02,406 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:02,409 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:02,410 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831782409 2024-11-17T08:23:02,418 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831782409 2024-11-17T08:23:02,424 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:23:02,426 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831782409, startPosition=0, beingWritten=true 2024-11-17T08:23:02,430 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831782409: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,433 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831782409, startPosition=197, beingWritten=true 2024-11-17T08:23:02,438 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831782409: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741875_1051 (size=317) 2024-11-17T08:23:02,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741849_1025 (size=468) 2024-11-17T08:23:02,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741875_1051 (size=317) 2024-11-17T08:23:02,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741875_1051 (size=317) 2024-11-17T08:23:02,456 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:02,456 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831782409) 2024-11-17T08:23:02,466 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=343 (was 339) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #66] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #66] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 428), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3257 (was 3263) 2024-11-17T08:23:02,477 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=343, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3257 2024-11-17T08:23:02,478 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:02,481 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:02,482 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 2024-11-17T08:23:02,489 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 2024-11-17T08:23:02,492 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:02,494 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482, startPosition=0, beingWritten=true 2024-11-17T08:23:02,500 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,502 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:23:02,504 DEBUG [Thread-682 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482, startPosition=0, beingWritten=true 2024-11-17T08:23:02,509 DEBUG [Thread-682 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,511 DEBUG [Thread-682 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:23:02,511 DEBUG [Thread-682 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 to pos 314, reset compression=false 2024-11-17T08:23:02,519 DEBUG [Thread-682 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-17T08:23:02,519 DEBUG [Thread-682 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 to pos 372, reset compression=false 2024-11-17T08:23:02,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741876_1052 (size=380) 2024-11-17T08:23:02,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741876_1052 (size=380) 2024-11-17T08:23:02,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741876_1052 (size=380) 2024-11-17T08:23:02,527 DEBUG [Thread-682 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 for the last time to pos 372, reset compression=false 2024-11-17T08:23:02,529 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:02,529 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831782482) 2024-11-17T08:23:02,534 INFO [Thread-682 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 2024-11-17T08:23:02,534 DEBUG [Thread-682 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 and length of the file is 380 2024-11-17T08:23:02,534 DEBUG [Thread-682 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731831782482 2024-11-17T08:23:02,535 DEBUG [Thread-682 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:23:02,535 DEBUG [Thread-682 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:23:02,540 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=349 (was 343) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:628) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-682 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy46.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$321/0x00007f6a147dc828.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) jdk.internal.reflect.GeneratedMethodAccessor30.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy47.getFileInfo(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) app//org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.getCurrentPathFileStatus(WALEntryStream.java:399) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.checkAllBytesParsed(WALEntryStream.java:417) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:315) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #71] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3250 (was 3257) 2024-11-17T08:23:02,550 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=349, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3249 2024-11-17T08:23:02,550 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:02,553 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:02,553 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 2024-11-17T08:23:02,560 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 2024-11-17T08:23:02,564 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:23:02,568 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782568 2024-11-17T08:23:02,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,577 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782568 2024-11-17T08:23:02,579 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:23:02,579 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 is not closed yet, will try archiving it next time 2024-11-17T08:23:02,579 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:02,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741877_1053 (size=93) 2024-11-17T08:23:02,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741877_1053 (size=93) 2024-11-17T08:23:02,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741877_1053 (size=93) 2024-11-17T08:23:02,581 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 2024-11-17T08:23:02,684 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 2024-11-17T08:23:02,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741879_1055 (size=85) 2024-11-17T08:23:02,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741879_1055 (size=85) 2024-11-17T08:23:02,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741879_1055 (size=85) 2024-11-17T08:23:02,697 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553, startPosition=0, beingWritten=false 2024-11-17T08:23:02,698 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 2024-11-17T08:23:02,700 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,701 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 for the last time to pos 0, reset compression=false 2024-11-17T08:23:02,703 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 2024-11-17T08:23:02,707 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 2024-11-17T08:23:02,708 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 and length of the file is 85 2024-11-17T08:23:02,708 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782553 2024-11-17T08:23:02,708 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782568, startPosition=0, beingWritten=true 2024-11-17T08:23:02,713 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731831782568: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741878_1054 (size=212) 2024-11-17T08:23:02,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741878_1054 (size=212) 2024-11-17T08:23:02,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741878_1054 (size=212) 2024-11-17T08:23:02,722 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:02,722 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831782568) 2024-11-17T08:23:02,730 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=354 (was 349) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #73] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3241 (was 3249) 2024-11-17T08:23:02,739 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=354, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3240 2024-11-17T08:23:02,739 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:02,742 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:02,742 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 2024-11-17T08:23:02,776 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 2024-11-17T08:23:02,783 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:02,785 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742, startPosition=0, beingWritten=true 2024-11-17T08:23:02,790 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,796 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-17T08:23:02,797 DEBUG [Thread-720 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742, startPosition=0, beingWritten=true 2024-11-17T08:23:02,805 DEBUG [Thread-720 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,807 DEBUG [Thread-720 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-17T08:23:02,807 DEBUG [Thread-720 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 to pos 322, reset compression=false 2024-11-17T08:23:02,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741880_1056 (size=330) 2024-11-17T08:23:02,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741880_1056 (size=330) 2024-11-17T08:23:02,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741880_1056 (size=330) 2024-11-17T08:23:02,814 INFO [Thread-720 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=322, currentPosition=330 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-17T08:23:02,814 DEBUG [Thread-720 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 for the last time to pos 322, reset compression=false 2024-11-17T08:23:02,814 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:02,814 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831782742) 2024-11-17T08:23:02,818 INFO [Thread-720 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 2024-11-17T08:23:02,821 INFO [Thread-720 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 was moved to hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 2024-11-17T08:23:02,822 DEBUG [Thread-720 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 and length of the file is 330 2024-11-17T08:23:02,822 DEBUG [Thread-720 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731831782742 2024-11-17T08:23:02,822 DEBUG [Thread-720 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-17T08:23:02,822 DEBUG [Thread-720 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-17T08:23:02,824 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=360 (was 354) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-720 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy46.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$351/0x00007f6a148a1db0.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) jdk.internal.reflect.GeneratedMethodAccessor32.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy47.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #74] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=429 (was 427) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3198 (was 3240) 2024-11-17T08:23:02,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=360, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3198 2024-11-17T08:23:02,833 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:02,836 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEmptyStream_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:02,837 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731831782837 2024-11-17T08:23:02,843 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731831782837 2024-11-17T08:23:02,845 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:02,849 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731831782837, startPosition=0, beingWritten=true 2024-11-17T08:23:02,854 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731831782837: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741881_1057 (size=93) 2024-11-17T08:23:02,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741881_1057 (size=93) 2024-11-17T08:23:02,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741881_1057 (size=93) 2024-11-17T08:23:02,862 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:02,862 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831782837) 2024-11-17T08:23:02,872 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=365 (was 360) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3196 (was 3198) 2024-11-17T08:23:02,882 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=365, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3195 2024-11-17T08:23:02,883 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:02,885 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:02,886 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886 2024-11-17T08:23:02,894 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886 2024-11-17T08:23:02,897 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:23:02,900 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886, startPosition=0, beingWritten=true 2024-11-17T08:23:02,906 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,908 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:02,908 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886, startPosition=204, beingWritten=true 2024-11-17T08:23:02,913 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,916 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782915 2024-11-17T08:23:02,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:02,923 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886 with entries=3, filesize=330 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782915 2024-11-17T08:23:02,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255)] 2024-11-17T08:23:02,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886 is not closed yet, will try archiving it next time 2024-11-17T08:23:02,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741882_1058 (size=338) 2024-11-17T08:23:02,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741882_1058 (size=338) 2024-11-17T08:23:02,926 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:02,926 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886, startPosition=267, beingWritten=true 2024-11-17T08:23:02,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741882_1058 (size=338) 2024-11-17T08:23:02,928 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:02,929 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:02,929 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782886 2024-11-17T08:23:03,029 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782915, startPosition=0, beingWritten=true 2024-11-17T08:23:03,034 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731831782915: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:03,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741883_1059 (size=212) 2024-11-17T08:23:03,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741883_1059 (size=212) 2024-11-17T08:23:03,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741883_1059 (size=212) 2024-11-17T08:23:03,041 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:03,042 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831782915) 2024-11-17T08:23:03,057 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=370 (was 365) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:34874 [Waiting for operation #75] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #79] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3189 (was 3195) 2024-11-17T08:23:03,067 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=370, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3189 2024-11-17T08:23:03,068 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:03,070 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:03,071 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831783071 2024-11-17T08:23:03,078 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831783071 2024-11-17T08:23:03,084 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:23:03,086 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831783071, startPosition=0, beingWritten=true 2024-11-17T08:23:03,091 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831783071: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:03,092 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831783071, startPosition=204, beingWritten=true 2024-11-17T08:23:03,096 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731831783071: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:03,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741884_1060 (size=322) 2024-11-17T08:23:03,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741884_1060 (size=322) 2024-11-17T08:23:03,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741884_1060 (size=322) 2024-11-17T08:23:03,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:03,109 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831783071) 2024-11-17T08:23:03,119 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=375 (was 370) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:48802 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #81] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3189 (was 3189) 2024-11-17T08:23:03,127 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=375, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=3189 2024-11-17T08:23:03,128 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-17T08:23:03,130 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs, maxLogs=32 2024-11-17T08:23:03,131 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783131 2024-11-17T08:23:03,138 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783131 2024-11-17T08:23:03,139 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45385:45385),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:38395:38395)] 2024-11-17T08:23:03,141 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783140 2024-11-17T08:23:03,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,147 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783131 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783140 2024-11-17T08:23:03,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38395:38395),(127.0.0.1/127.0.0.1:38255:38255),(127.0.0.1/127.0.0.1:45385:45385)] 2024-11-17T08:23:03,149 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783131 is not closed yet, will try archiving it next time 2024-11-17T08:23:03,149 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T08:23:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741885_1061 (size=212) 2024-11-17T08:23:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741885_1061 (size=212) 2024-11-17T08:23:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741885_1061 (size=212) 2024-11-17T08:23:03,251 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783131, startPosition=0, beingWritten=false 2024-11-17T08:23:03,254 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783131: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-17T08:23:03,256 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:33161/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731831783131 2024-11-17T08:23:03,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T08:23:03,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44055 is added to blk_1073741886_1062 (size=93) 2024-11-17T08:23:03,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46265 is added to blk_1073741886_1062 (size=93) 2024-11-17T08:23:03,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45541 is added to blk_1073741886_1062 (size=93) 2024-11-17T08:23:03,264 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/oldWALs 2024-11-17T08:23:03,264 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1731831783140) 2024-11-17T08:23:03,273 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=380 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1179087624_22 at /127.0.0.1:57694 [Waiting for operation #83] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=3184 (was 3189) 2024-11-17T08:23:03,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T08:23:03,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1738cb2d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T08:23:03,280 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1473b53e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T08:23:03,280 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T08:23:03,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b9fb405{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T08:23:03,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c4dfe8c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir/,STOPPED} 2024-11-17T08:23:03,283 WARN [BP-2071561254-172.17.0.3-1731831769523 heartbeating to localhost/127.0.0.1:33161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T08:23:03,283 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T08:23:03,283 WARN [BP-2071561254-172.17.0.3-1731831769523 heartbeating to localhost/127.0.0.1:33161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2071561254-172.17.0.3-1731831769523 (Datanode Uuid f36dad06-3263-4aa1-89b9-ad3b577b967f) service to localhost/127.0.0.1:33161 2024-11-17T08:23:03,283 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T08:23:03,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data5/current/BP-2071561254-172.17.0.3-1731831769523 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T08:23:03,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data6/current/BP-2071561254-172.17.0.3-1731831769523 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T08:23:03,286 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T08:23:03,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a6c60e4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T08:23:03,288 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@41e0aaee{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T08:23:03,288 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T08:23:03,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2544e7fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T08:23:03,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75fd415{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir/,STOPPED} 2024-11-17T08:23:03,289 WARN [BP-2071561254-172.17.0.3-1731831769523 heartbeating to localhost/127.0.0.1:33161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T08:23:03,289 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T08:23:03,289 WARN [BP-2071561254-172.17.0.3-1731831769523 heartbeating to localhost/127.0.0.1:33161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2071561254-172.17.0.3-1731831769523 (Datanode Uuid 1db8413a-aa16-40d9-a659-93add888ceb8) service to localhost/127.0.0.1:33161 2024-11-17T08:23:03,289 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T08:23:03,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data3/current/BP-2071561254-172.17.0.3-1731831769523 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T08:23:03,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data4/current/BP-2071561254-172.17.0.3-1731831769523 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T08:23:03,291 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T08:23:03,293 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7ec85977{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T08:23:03,294 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e913c20{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T08:23:03,294 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T08:23:03,294 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ca42ed7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T08:23:03,294 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b53287d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir/,STOPPED} 2024-11-17T08:23:03,297 WARN [BP-2071561254-172.17.0.3-1731831769523 heartbeating to localhost/127.0.0.1:33161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T08:23:03,297 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T08:23:03,297 WARN [BP-2071561254-172.17.0.3-1731831769523 heartbeating to localhost/127.0.0.1:33161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2071561254-172.17.0.3-1731831769523 (Datanode Uuid 4c783461-adb2-4e93-a110-b43275127ba1) service to localhost/127.0.0.1:33161 2024-11-17T08:23:03,297 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T08:23:03,298 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data1/current/BP-2071561254-172.17.0.3-1731831769523 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T08:23:03,298 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/cluster_100de09c-5dd8-0e78-f9ed-29f7b451a767/data/data2/current/BP-2071561254-172.17.0.3-1731831769523 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T08:23:03,299 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T08:23:03,307 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44567ea4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T08:23:03,307 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@216da5e5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T08:23:03,307 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T08:23:03,307 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a9b22e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T08:23:03,308 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c2d1933{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f94c668c-7dae-820d-7fa5-7190d4cbb23a/hadoop.log.dir/,STOPPED} 2024-11-17T08:23:03,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down