2024-11-10 01:23:58,349 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-10 01:23:58,360 main DEBUG Took 0.009331 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 01:23:58,360 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 01:23:58,361 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 01:23:58,362 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 01:23:58,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,371 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 01:23:58,383 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,385 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,385 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,386 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,386 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,387 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,388 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,388 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,389 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,390 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,390 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,390 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,391 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,391 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,392 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,392 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,392 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,393 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,393 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,393 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,394 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,394 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 01:23:58,395 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,395 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 01:23:58,396 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 01:23:58,398 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 01:23:58,400 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 01:23:58,400 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 01:23:58,401 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 01:23:58,402 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 01:23:58,410 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 01:23:58,412 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 01:23:58,414 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 01:23:58,415 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 01:23:58,415 main DEBUG createAppenders(={Console}) 2024-11-10 01:23:58,416 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-10 01:23:58,416 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-10 01:23:58,416 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-10 01:23:58,417 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 01:23:58,417 main DEBUG OutputStream closed 2024-11-10 01:23:58,418 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 01:23:58,418 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 01:23:58,418 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2dc54ad4 OK 2024-11-10 01:23:58,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 01:23:58,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 01:23:58,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 01:23:58,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 01:23:58,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 01:23:58,493 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 01:23:58,493 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 01:23:58,494 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 01:23:58,494 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 01:23:58,494 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 01:23:58,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 01:23:58,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 01:23:58,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 01:23:58,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 01:23:58,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 01:23:58,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 01:23:58,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 01:23:58,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 01:23:58,500 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 01:23:58,501 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4f9a2c08) with optional ClassLoader: null 2024-11-10 01:23:58,501 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 01:23:58,502 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4f9a2c08] started OK. 2024-11-10T01:23:58,719 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92 2024-11-10 01:23:58,722 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 01:23:58,723 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T01:23:58,769 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStreamFSHLog timeout: 26 mins 2024-11-10T01:23:58,803 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145, deleteOnExit=true 2024-11-10T01:23:58,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/test.cache.data in system properties and HBase conf 2024-11-10T01:23:58,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T01:23:58,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir in system properties and HBase conf 2024-11-10T01:23:58,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T01:23:58,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T01:23:58,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T01:23:58,901 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T01:23:58,998 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T01:23:59,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T01:23:59,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T01:23:59,006 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T01:23:59,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T01:23:59,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T01:23:59,008 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T01:23:59,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T01:23:59,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T01:23:59,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T01:23:59,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/nfs.dump.dir in system properties and HBase conf 2024-11-10T01:23:59,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/java.io.tmpdir in system properties and HBase conf 2024-11-10T01:23:59,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T01:23:59,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T01:23:59,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T01:24:00,372 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T01:24:00,472 INFO [Time-limited test {}] log.Log(170): Logging initialized @3151ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T01:24:00,571 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T01:24:00,659 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T01:24:00,687 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T01:24:00,688 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T01:24:00,690 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T01:24:00,709 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T01:24:00,713 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c2d1933{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir/,AVAILABLE} 2024-11-10T01:24:00,714 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a9b22e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T01:24:00,955 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44567ea4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/java.io.tmpdir/jetty-localhost-39271-hadoop-hdfs-3_4_1-tests_jar-_-any-3992161121954681459/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T01:24:00,977 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@216da5e5{HTTP/1.1, (http/1.1)}{localhost:39271} 2024-11-10T01:24:00,978 INFO [Time-limited test {}] server.Server(415): Started @3657ms 2024-11-10T01:24:01,800 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T01:24:01,810 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T01:24:01,811 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T01:24:01,812 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T01:24:01,812 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T01:24:01,813 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@309d3e97{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir/,AVAILABLE} 2024-11-10T01:24:01,814 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d2a62e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T01:24:01,916 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5a86aa33{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/java.io.tmpdir/jetty-localhost-44529-hadoop-hdfs-3_4_1-tests_jar-_-any-13086838892306734436/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T01:24:01,917 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53cf346{HTTP/1.1, (http/1.1)}{localhost:44529} 2024-11-10T01:24:01,917 INFO [Time-limited test {}] server.Server(415): Started @4597ms 2024-11-10T01:24:01,964 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T01:24:02,071 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T01:24:02,079 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T01:24:02,080 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T01:24:02,080 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T01:24:02,081 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T01:24:02,082 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cd634bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir/,AVAILABLE} 2024-11-10T01:24:02,082 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45d510a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T01:24:02,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@fecc8a3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/java.io.tmpdir/jetty-localhost-46559-hadoop-hdfs-3_4_1-tests_jar-_-any-3549567033652997917/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T01:24:02,198 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@cbbb804{HTTP/1.1, (http/1.1)}{localhost:46559} 2024-11-10T01:24:02,199 INFO [Time-limited test {}] server.Server(415): Started @4879ms 2024-11-10T01:24:02,201 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T01:24:02,251 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T01:24:02,256 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T01:24:02,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T01:24:02,263 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T01:24:02,263 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T01:24:02,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f179af2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir/,AVAILABLE} 2024-11-10T01:24:02,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dfa9f68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T01:24:02,388 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d954ade{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/java.io.tmpdir/jetty-localhost-43191-hadoop-hdfs-3_4_1-tests_jar-_-any-5660441537161425111/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T01:24:02,389 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@76890781{HTTP/1.1, (http/1.1)}{localhost:43191} 2024-11-10T01:24:02,390 INFO [Time-limited test {}] server.Server(415): Started @5069ms 2024-11-10T01:24:02,393 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T01:24:03,871 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data2/current/BP-1442331070-172.17.0.3-1731201839759/current, will proceed with Du for space computation calculation, 2024-11-10T01:24:03,871 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data1/current/BP-1442331070-172.17.0.3-1731201839759/current, will proceed with Du for space computation calculation, 2024-11-10T01:24:03,896 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T01:24:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x46c2ba8014502a53 with lease ID 0x38009f7e2cac3461: Processing first storage report for DS-a3b9e0e8-25b6-446c-8f4e-797edbf61e92 from datanode DatanodeRegistration(127.0.0.1:46541, datanodeUuid=d3ce9723-82d4-40c3-86ff-701dbee99167, infoPort=39017, infoSecurePort=0, ipcPort=44333, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759) 2024-11-10T01:24:03,942 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x46c2ba8014502a53 with lease ID 0x38009f7e2cac3461: from storage DS-a3b9e0e8-25b6-446c-8f4e-797edbf61e92 node DatanodeRegistration(127.0.0.1:46541, datanodeUuid=d3ce9723-82d4-40c3-86ff-701dbee99167, infoPort=39017, infoSecurePort=0, ipcPort=44333, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T01:24:03,942 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x46c2ba8014502a53 with lease ID 0x38009f7e2cac3461: Processing first storage report for DS-51fc5dcd-fc31-4948-b1bb-28d4a6dbbb77 from datanode DatanodeRegistration(127.0.0.1:46541, datanodeUuid=d3ce9723-82d4-40c3-86ff-701dbee99167, infoPort=39017, infoSecurePort=0, ipcPort=44333, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759) 2024-11-10T01:24:03,942 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x46c2ba8014502a53 with lease ID 0x38009f7e2cac3461: from storage DS-51fc5dcd-fc31-4948-b1bb-28d4a6dbbb77 node DatanodeRegistration(127.0.0.1:46541, datanodeUuid=d3ce9723-82d4-40c3-86ff-701dbee99167, infoPort=39017, infoSecurePort=0, ipcPort=44333, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T01:24:04,091 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data4/current/BP-1442331070-172.17.0.3-1731201839759/current, will proceed with Du for space computation calculation, 2024-11-10T01:24:04,091 WARN [Thread-131 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data3/current/BP-1442331070-172.17.0.3-1731201839759/current, will proceed with Du for space computation calculation, 2024-11-10T01:24:04,116 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T01:24:04,121 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8cb33bf492281118 with lease ID 0x38009f7e2cac3462: Processing first storage report for DS-33dde4ab-0f80-433f-9d2d-ebb7dc7d81d1 from datanode DatanodeRegistration(127.0.0.1:44509, datanodeUuid=7e197415-7c30-4373-9882-20e2ea42017d, infoPort=38203, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759) 2024-11-10T01:24:04,122 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8cb33bf492281118 with lease ID 0x38009f7e2cac3462: from storage DS-33dde4ab-0f80-433f-9d2d-ebb7dc7d81d1 node DatanodeRegistration(127.0.0.1:44509, datanodeUuid=7e197415-7c30-4373-9882-20e2ea42017d, infoPort=38203, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T01:24:04,122 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8cb33bf492281118 with lease ID 0x38009f7e2cac3462: Processing first storage report for DS-f81bd3ff-f769-4776-a358-4408b58bad70 from datanode DatanodeRegistration(127.0.0.1:44509, datanodeUuid=7e197415-7c30-4373-9882-20e2ea42017d, infoPort=38203, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759) 2024-11-10T01:24:04,122 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8cb33bf492281118 with lease ID 0x38009f7e2cac3462: from storage DS-f81bd3ff-f769-4776-a358-4408b58bad70 node DatanodeRegistration(127.0.0.1:44509, datanodeUuid=7e197415-7c30-4373-9882-20e2ea42017d, infoPort=38203, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T01:24:04,284 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data5/current/BP-1442331070-172.17.0.3-1731201839759/current, will proceed with Du for space computation calculation, 2024-11-10T01:24:04,284 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data6/current/BP-1442331070-172.17.0.3-1731201839759/current, will proceed with Du for space computation calculation, 2024-11-10T01:24:04,304 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T01:24:04,309 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc13f70d022290ab3 with lease ID 0x38009f7e2cac3463: Processing first storage report for DS-e85ff591-b0a1-4039-94c6-01f7dcc66203 from datanode DatanodeRegistration(127.0.0.1:35505, datanodeUuid=d75c62cb-52bb-40d0-bbce-3fbb4e8edd67, infoPort=36557, infoSecurePort=0, ipcPort=41529, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759) 2024-11-10T01:24:04,309 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc13f70d022290ab3 with lease ID 0x38009f7e2cac3463: from storage DS-e85ff591-b0a1-4039-94c6-01f7dcc66203 node DatanodeRegistration(127.0.0.1:35505, datanodeUuid=d75c62cb-52bb-40d0-bbce-3fbb4e8edd67, infoPort=36557, infoSecurePort=0, ipcPort=41529, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T01:24:04,309 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc13f70d022290ab3 with lease ID 0x38009f7e2cac3463: Processing first storage report for DS-37d3e17f-8d8b-4e94-b936-b3050a087b5d from datanode DatanodeRegistration(127.0.0.1:35505, datanodeUuid=d75c62cb-52bb-40d0-bbce-3fbb4e8edd67, infoPort=36557, infoSecurePort=0, ipcPort=41529, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759) 2024-11-10T01:24:04,309 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc13f70d022290ab3 with lease ID 0x38009f7e2cac3463: from storage DS-37d3e17f-8d8b-4e94-b936-b3050a087b5d node DatanodeRegistration(127.0.0.1:35505, datanodeUuid=d75c62cb-52bb-40d0-bbce-3fbb4e8edd67, infoPort=36557, infoSecurePort=0, ipcPort=41529, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T01:24:04,364 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92 2024-11-10T01:24:04,380 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=155, OpenFileDescriptor=393, MaxFileDescriptor=1048576, SystemLoadAverage=556, ProcessCount=11, AvailableMemoryMB=2732 2024-11-10T01:24:04,383 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T01:24:04,973 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T01:24:04,992 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:05,066 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:05,106 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201845099 2024-11-10T01:24:05,328 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201845099 2024-11-10T01:24:05,380 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:05,440 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201845099, startPosition=0, beingWritten=true 2024-11-10T01:24:05,580 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_0_ isCompressionEnabled=false]/testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201845099 to pos 203, reset compression=false 2024-11-10T01:24:05,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:05,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:05,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:05,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:05,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:05,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741825_1001 (size=447) 2024-11-10T01:24:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741825_1001 (size=447) 2024-11-10T01:24:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741825_1001 (size=447) 2024-11-10T01:24:06,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:06,039 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201845099) 2024-11-10T01:24:06,050 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[0: isCompressionEnabled=false] Thread=166 (was 155) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44367 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@67f79008 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=411 (was 393) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 556) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2588 (was 2732) 2024-11-10T01:24:06,060 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=166, OpenFileDescriptor=411, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2586 2024-11-10T01:24:06,062 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:06,069 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:06,071 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071 2024-11-10T01:24:06,094 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071 2024-11-10T01:24:06,098 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:06,102 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071, startPosition=0, beingWritten=true 2024-11-10T01:24:06,209 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:06,213 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071, startPosition=0, beingWritten=true 2024-11-10T01:24:06,233 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:06,259 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071, startPosition=0, beingWritten=true 2024-11-10T01:24:06,271 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:06,303 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071, startPosition=0, beingWritten=true 2024-11-10T01:24:06,323 WARN [Thread-175 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:06,364 DEBUG [Thread-175 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071, startPosition=0, beingWritten=true 2024-11-10T01:24:06,379 DEBUG [Thread-175 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:06,379 DEBUG [Thread-175 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071 to pos 460, reset compression=false 2024-11-10T01:24:06,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741826_1002 (size=468) 2024-11-10T01:24:06,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741826_1002 (size=468) 2024-11-10T01:24:06,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741826_1002 (size=468) 2024-11-10T01:24:06,409 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:06,410 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201846071) 2024-11-10T01:24:06,410 DEBUG [Thread-175 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071 to pos 460, reset compression=false 2024-11-10T01:24:06,423 INFO [Thread-175 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071 2024-11-10T01:24:06,429 DEBUG [Thread-175 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201846071 2024-11-10T01:24:06,430 DEBUG [Thread-175 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:06,430 DEBUG [Thread-175 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:06,439 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[0: isCompressionEnabled=false] Thread=173 (was 166) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-175 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy46.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getBlockLocations$0(ClientNamenodeProtocolTranslatorPB.java:340) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB$$Lambda$337/0x00007f7edc878fd0.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:340) java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) app/jdk.proxy2/jdk.proxy2.$Proxy47.getBlockLocations(Unknown Source) app//org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:931) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:920) app//org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:909) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1076) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=415 (was 411) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2536 (was 2586) 2024-11-10T01:24:06,448 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=173, OpenFileDescriptor=415, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2534 2024-11-10T01:24:06,449 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:06,455 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:06,458 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1731201846458 2024-11-10T01:24:06,475 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D.1731201846458 2024-11-10T01:24:06,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:06,587 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.false 2024-11-10T01:24:06,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741828_1004 (size=231) 2024-11-10T01:24:06,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741828_1004 (size=231) 2024-11-10T01:24:06,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741828_1004 (size=231) 2024-11-10T01:24:06,605 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-10T01:24:06,606 DEBUG [Thread-199 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/user/jenkins/log.1.false, startPosition=0, beingWritten=false 2024-11-10T01:24:06,613 DEBUG [Thread-199 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/user/jenkins/log.1.false 2024-11-10T01:24:06,613 DEBUG [Thread-199 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:06,614 DEBUG [Thread-199 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/user/jenkins/log.2.false, startPosition=0, beingWritten=false 2024-11-10T01:24:06,616 WARN [Thread-199 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:44367/user/jenkins/log.2.false, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-10T01:24:06,616 DEBUG [Thread-199 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/user/jenkins/log.2.false 2024-11-10T01:24:06,617 DEBUG [Thread-199 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:06,617 DEBUG [Thread-199 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:06,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741827_1003 (size=93) 2024-11-10T01:24:06,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741827_1003 (size=93) 2024-11-10T01:24:06,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741827_1003 (size=93) 2024-11-10T01:24:06,627 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:06,628 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201846458) 2024-11-10T01:24:06,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[0: isCompressionEnabled=false] Thread=178 (was 173) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 415) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2513 (was 2534) 2024-11-10T01:24:06,644 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=178, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2513 2024-11-10T01:24:06,645 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:06,650 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:06,652 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 2024-11-10T01:24:06,662 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 2024-11-10T01:24:06,663 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:06,665 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 2024-11-10T01:24:06,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,677 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 2024-11-10T01:24:06,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741829_1005 (size=93) 2024-11-10T01:24:06,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741829_1005 (size=93) 2024-11-10T01:24:06,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741829_1005 (size=93) 2024-11-10T01:24:06,682 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:06,683 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 2024-11-10T01:24:06,684 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:06,686 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 2024-11-10T01:24:06,697 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:06,698 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:06,699 DEBUG [Thread-216 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652, startPosition=0, beingWritten=false 2024-11-10T01:24:06,701 INFO [Thread-216 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 2024-11-10T01:24:06,703 WARN [Thread-216 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-10T01:24:06,703 DEBUG [Thread-216 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846652 2024-11-10T01:24:06,704 DEBUG [Thread-216 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665, startPosition=0, beingWritten=true 2024-11-10T01:24:06,723 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 to pos 0, reset compression=false 2024-11-10T01:24:06,743 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 to pos 0, reset compression=false 2024-11-10T01:24:06,765 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 to pos 0, reset compression=false 2024-11-10T01:24:06,785 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 to pos 0, reset compression=false 2024-11-10T01:24:06,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,803 DEBUG [Thread-216 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 to pos 0, reset compression=false 2024-11-10T01:24:06,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741830_1006 (size=93) 2024-11-10T01:24:06,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741830_1006 (size=93) 2024-11-10T01:24:06,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741830_1006 (size=93) 2024-11-10T01:24:06,812 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:06,812 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201846665) 2024-11-10T01:24:06,815 INFO [Thread-216 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 2024-11-10T01:24:06,819 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[0: isCompressionEnabled=false] Thread=184 (was 178) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-216 java.base@17.0.11/java.util.Formatter$FormatSpecifier.printString(Formatter.java:3051) java.base@17.0.11/java.util.Formatter$FormatSpecifier.print(Formatter.java:2933) java.base@17.0.11/java.util.Formatter.format(Formatter.java:2689) java.base@17.0.11/java.util.Formatter.format(Formatter.java:2625) java.base@17.0.11/java.lang.String.format(String.java:4145) app//org.apache.hadoop.fs.FileSystem.get(FileSystem.java:551) app//org.apache.hadoop.fs.FileSystem.get(FileSystem.java:289) app//org.apache.hadoop.fs.FileSystem.get(FileSystem.java:541) app//org.apache.hadoop.fs.Path.getFileSystem(Path.java:366) app//org.apache.hadoop.hbase.util.CommonFSUtils.getRootDir(CommonFSUtils.java:292) app//org.apache.hadoop.hbase.util.CommonFSUtils.isValidWALRootDir(CommonFSUtils.java:358) app//org.apache.hadoop.hbase.util.CommonFSUtils.getWALRootDir(CommonFSUtils.java:322) app//org.apache.hadoop.hbase.wal.AbstractFSWALProvider.findArchivedLog(AbstractFSWALProvider.java:529) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.openArchivedWAL(AbstractProtobufWALReader.java:183) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:196) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:212) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2502 (was 2513) 2024-11-10T01:24:06,821 DEBUG [Thread-216 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_0_ isCompressionEnabled=false]/testEOFExceptionInOldWALsDirectory_0_+isCompressionEnabled%3Dfalse%5D.1731201846665 2024-11-10T01:24:06,822 DEBUG [Thread-216 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:06,822 DEBUG [Thread-216 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:06,826 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=184, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2502 2024-11-10T01:24:06,828 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:06,833 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:06,835 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846835 2024-11-10T01:24:06,846 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846835 2024-11-10T01:24:06,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:06,853 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846852 2024-11-10T01:24:06,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,872 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846835 with entries=10, filesize=1.30 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846852 2024-11-10T01:24:06,873 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:06,873 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846835 is not closed yet, will try archiving it next time 2024-11-10T01:24:06,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741831_1007 (size=1343) 2024-11-10T01:24:06,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741831_1007 (size=1343) 2024-11-10T01:24:06,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741831_1007 (size=1343) 2024-11-10T01:24:06,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741832_1008 (size=718) 2024-11-10T01:24:06,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741832_1008 (size=718) 2024-11-10T01:24:06,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741832_1008 (size=718) 2024-11-10T01:24:06,884 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-10T01:24:06,885 DEBUG [Thread-239 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846835, startPosition=0, beingWritten=false 2024-11-10T01:24:06,892 DEBUG [Thread-239 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-10T01:24:06,893 DEBUG [Thread-239 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846835 2024-11-10T01:24:06,893 DEBUG [Thread-239 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846852, startPosition=0, beingWritten=false 2024-11-10T01:24:06,902 DEBUG [Thread-239 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D.1731201846852 2024-11-10T01:24:06,902 DEBUG [Thread-239 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-10T01:24:06,903 DEBUG [Thread-239 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:06,903 DEBUG [Thread-239 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:06,908 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:06,908 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201846852) 2024-11-10T01:24:06,916 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[0: isCompressionEnabled=false] Thread=189 (was 184) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2498 (was 2502) 2024-11-10T01:24:06,924 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=189, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2498 2024-11-10T01:24:06,925 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:06,930 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:06,931 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201846931 2024-11-10T01:24:06,934 WARN [IPC Server handler 0 on default port 44367 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T01:24:06,934 WARN [IPC Server handler 0 on default port 44367 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T01:24:06,934 WARN [IPC Server handler 0 on default port 44367 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T01:24:06,940 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionForRecoveredQueue_0_ isCompressionEnabled=false]/testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201846931 2024-11-10T01:24:06,942 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:06,950 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-10T01:24:06,952 DEBUG [Thread-247 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-10T01:24:06,954 WARN [Thread-247 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-10T01:24:06,955 DEBUG [Thread-247 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-10T01:24:06,955 DEBUG [Thread-247 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:06,955 DEBUG [Thread-247 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:06,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:06,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741833_1009 (size=93) 2024-11-10T01:24:06,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741833_1009 (size=93) 2024-11-10T01:24:06,964 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:06,964 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201846931) 2024-11-10T01:24:06,971 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[0: isCompressionEnabled=false] Thread=194 (was 189) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2498 (was 2498) 2024-11-10T01:24:06,978 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=194, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2498 2024-11-10T01:24:06,980 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:06,984 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:06,985 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846985 2024-11-10T01:24:06,988 WARN [IPC Server handler 4 on default port 44367 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T01:24:06,988 WARN [IPC Server handler 4 on default port 44367 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T01:24:06,988 WARN [IPC Server handler 4 on default port 44367 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T01:24:06,994 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846985 2024-11-10T01:24:06,995 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:06,997 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846997 2024-11-10T01:24:07,001 WARN [IPC Server handler 2 on default port 44367 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T01:24:07,002 WARN [IPC Server handler 2 on default port 44367 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T01:24:07,002 WARN [IPC Server handler 2 on default port 44367 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T01:24:07,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,010 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846985 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846997 2024-11-10T01:24:07,011 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:07,011 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846985 is not closed yet, will try archiving it next time 2024-11-10T01:24:07,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741834_1010 (size=218) 2024-11-10T01:24:07,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741834_1010 (size=218) 2024-11-10T01:24:07,018 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:07,021 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:07,022 DEBUG [Thread-260 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846985, startPosition=0, beingWritten=false 2024-11-10T01:24:07,028 DEBUG [Thread-260 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846985 2024-11-10T01:24:07,028 DEBUG [Thread-260 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-10T01:24:07,029 DEBUG [Thread-260 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846997, startPosition=0, beingWritten=true 2024-11-10T01:24:07,048 DEBUG [Thread-260 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-10T01:24:07,048 DEBUG [Thread-260 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846997 to pos 2585, reset compression=false 2024-11-10T01:24:07,048 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201847048 2024-11-10T01:24:07,051 WARN [IPC Server handler 1 on default port 44367 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T01:24:07,052 WARN [IPC Server handler 1 on default port 44367 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T01:24:07,052 WARN [IPC Server handler 1 on default port 44367 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T01:24:07,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,063 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846997 with entries=20, filesize=2.52 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201847048 2024-11-10T01:24:07,065 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:07,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741835_1011 (size=2593) 2024-11-10T01:24:07,066 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846997 is not closed yet, will try archiving it next time 2024-11-10T01:24:07,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741835_1011 (size=2593) 2024-11-10T01:24:07,086 DEBUG [Thread-260 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846997 to pos 2585, reset compression=false 2024-11-10T01:24:07,090 DEBUG [Thread-260 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201846997 2024-11-10T01:24:07,091 DEBUG [Thread-260 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201847048, startPosition=0, beingWritten=true 2024-11-10T01:24:07,102 DEBUG [Thread-260 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-10T01:24:07,103 DEBUG [Thread-260 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201847048 to pos 1335, reset compression=false 2024-11-10T01:24:07,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741836_1012 (size=1343) 2024-11-10T01:24:07,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741836_1012 (size=1343) 2024-11-10T01:24:07,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:07,119 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201847048) 2024-11-10T01:24:07,121 DEBUG [Thread-260 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWrongPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201847048 2024-11-10T01:24:07,122 DEBUG [Thread-260 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:07,122 DEBUG [Thread-260 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:07,127 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[0: isCompressionEnabled=false] Thread=200 (was 194) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-260 app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.trimIdleSelectors(SocketIOWithTimeout.java:432) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.release(SocketIOWithTimeout.java:413) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:374) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.newBlockReader(BlockReaderRemote.java:419) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:861) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:757) app//org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:381) app//org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:715) app//org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:645) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:845) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.DataInputStream.readInt(DataInputStream.java:381) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:628) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2488 (was 2498) 2024-11-10T01:24:07,137 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=200, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2486 2024-11-10T01:24:07,138 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:07,143 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALKeySerialization_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:07,145 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1731201847144 2024-11-10T01:24:07,161 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALKeySerialization_0_ isCompressionEnabled=false]/testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D.1731201847144 2024-11-10T01:24:07,165 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:07,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741837_1013 (size=93) 2024-11-10T01:24:07,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741837_1013 (size=93) 2024-11-10T01:24:07,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741837_1013 (size=93) 2024-11-10T01:24:07,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:07,177 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201847144) 2024-11-10T01:24:07,185 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[0: isCompressionEnabled=false] Thread=205 (was 200) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2483 (was 2486) 2024-11-10T01:24:07,192 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=205, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2483 2024-11-10T01:24:07,193 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:07,198 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:07,199 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731201847199 2024-11-10T01:24:07,215 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731201847199 2024-11-10T01:24:07,221 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:07,231 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731201847199, startPosition=0, beingWritten=true 2024-11-10T01:24:07,243 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:07,268 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:07,374 DEBUG [Thread-288 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731201847199, startPosition=0, beingWritten=true 2024-11-10T01:24:07,383 DEBUG [Thread-288 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:07,383 DEBUG [Thread-288 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731201847199 to pos 460, reset compression=false 2024-11-10T01:24:07,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741838_1014 (size=468) 2024-11-10T01:24:07,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741838_1014 (size=468) 2024-11-10T01:24:07,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741838_1014 (size=468) 2024-11-10T01:24:07,395 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:07,395 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201847199) 2024-11-10T01:24:07,400 INFO [Thread-288 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731201847199 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731201847199 2024-11-10T01:24:07,404 DEBUG [Thread-288 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderDisabled_0_+isCompressionEnabled%3Dfalse%5D.1731201847199 2024-11-10T01:24:07,405 DEBUG [Thread-288 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:07,405 DEBUG [Thread-288 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:07,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[0: isCompressionEnabled=false] Thread=214 (was 205) Potentially hanging thread: IPC Client (597607644) connection to localhost/127.0.0.1:41529 from blk_1073741838_1014 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:132) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:518) java.base@17.0.11/java.io.DataInputStream.readInt(DataInputStream.java:381) app//org.apache.hadoop.ipc.Client$IpcStreams.readResponse(Client.java:1923) app//org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1203) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1094) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-288 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:65) app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) app//org.apache.hadoop.ipc.Client.call(Client.java:1529) app//org.apache.hadoop.ipc.Client.call(Client.java:1426) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) app/jdk.proxy2/jdk.proxy2.$Proxy85.getReplicaVisibleLength(Unknown Source) app//org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB.lambda$getReplicaVisibleLength$0(ClientDatanodeProtocolTranslatorPB.java:199) app//org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB$$Lambda$339/0x00007f7edc894e10.call(Unknown Source) app//org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) app//org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB.getReplicaVisibleLength(ClientDatanodeProtocolTranslatorPB.java:199) app//org.apache.hadoop.hdfs.DFSInputStream.readBlockLength(DFSInputStream.java:352) app//org.apache.hadoop.hdfs.DFSInputStream.getLastBlockLength(DFSInputStream.java:323) app//org.apache.hadoop.hdfs.DFSInputStream.openInfo(DFSInputStream.java:243) app//org.apache.hadoop.hdfs.DFSInputStream.(DFSInputStream.java:213) app//org.apache.hadoop.hdfs.DFSClient.openInternal(DFSClient.java:1114) app//org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1077) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:342) app//org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338) app//org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) app//org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:355) app//org.apache.hadoop.fs.FileSystem.open(FileSystem.java:997) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.open(AbstractProtobufWALReader.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:272) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:225) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2434 (was 2483) 2024-11-10T01:24:07,413 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=213, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2433 2024-11-10T01:24:07,414 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:07,417 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:07,418 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847418 2024-11-10T01:24:07,427 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847418 2024-11-10T01:24:07,428 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:07,432 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:07,432 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847418, startPosition=0, beingWritten=true 2024-11-10T01:24:07,440 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847440 2024-11-10T01:24:07,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,452 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847418 with entries=3, filesize=439 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847440 2024-11-10T01:24:07,454 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:07,454 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847418 is not closed yet, will try archiving it next time 2024-11-10T01:24:07,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741839_1015 (size=447) 2024-11-10T01:24:07,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741839_1015 (size=447) 2024-11-10T01:24:07,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741839_1015 (size=447) 2024-11-10T01:24:07,456 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:07,457 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:07,457 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847418 for the last time to pos 321, reset compression=false 2024-11-10T01:24:07,462 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:07,462 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847418 2024-11-10T01:24:07,563 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_0_ isCompressionEnabled=false]/testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D.1731201847440, startPosition=0, beingWritten=true 2024-11-10T01:24:07,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741840_1016 (size=211) 2024-11-10T01:24:07,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741840_1016 (size=211) 2024-11-10T01:24:07,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741840_1016 (size=211) 2024-11-10T01:24:07,593 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:07,593 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201847440) 2024-11-10T01:24:07,606 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[0: isCompressionEnabled=false] Thread=218 (was 213) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2423 (was 2433) 2024-11-10T01:24:07,616 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=218, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2403 2024-11-10T01:24:07,617 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:07,622 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:07,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847623 2024-11-10T01:24:07,638 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847623 2024-11-10T01:24:07,643 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:07,647 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:07,648 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847623, startPosition=0, beingWritten=true 2024-11-10T01:24:07,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847658 2024-11-10T01:24:07,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,675 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847623 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847658 2024-11-10T01:24:07,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741841_1017 (size=218) 2024-11-10T01:24:07,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741841_1017 (size=218) 2024-11-10T01:24:07,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741841_1017 (size=218) 2024-11-10T01:24:07,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:07,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847623 is not closed yet, will try archiving it next time 2024-11-10T01:24:07,682 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:07,683 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847623 for the last time to pos 210, reset compression=false 2024-11-10T01:24:07,688 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847623 and length of the file is 218 2024-11-10T01:24:07,688 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847623 2024-11-10T01:24:07,789 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_0_ isCompressionEnabled=false]/testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D.1731201847658, startPosition=0, beingWritten=true 2024-11-10T01:24:07,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:07,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741842_1018 (size=218) 2024-11-10T01:24:07,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741842_1018 (size=218) 2024-11-10T01:24:07,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741842_1018 (size=218) 2024-11-10T01:24:07,812 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:07,812 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201847658) 2024-11-10T01:24:07,820 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[0: isCompressionEnabled=false] Thread=223 (was 218) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2388 (was 2403) 2024-11-10T01:24:07,827 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=223, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2387 2024-11-10T01:24:07,828 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:07,832 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:07,833 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731201847833 2024-11-10T01:24:07,843 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731201847833 2024-11-10T01:24:07,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:07,853 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731201847833, startPosition=0, beingWritten=true 2024-11-10T01:24:07,862 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-10T01:24:08,863 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731201847833 to pos 203, reset compression=false 2024-11-10T01:24:08,876 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 272 and read up to 320 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.KeyValueUtil.createKeyValueFromInputStream(KeyValueUtil.java:659) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.KeyValueCodecWithTags$KeyValueDecoder.parseCell(KeyValueCodecWithTags.java:77) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-10T01:24:08,877 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_0_ isCompressionEnabled=false]/testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D.1731201847833 to pos 203, reset compression=false 2024-11-10T01:24:08,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741843_1019 (size=329) 2024-11-10T01:24:08,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741843_1019 (size=329) 2024-11-10T01:24:08,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741843_1019 (size=329) 2024-11-10T01:24:08,897 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:08,898 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201847833) 2024-11-10T01:24:08,905 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[0: isCompressionEnabled=false] Thread=228 (was 223) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2344 (was 2387) 2024-11-10T01:24:08,912 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=228, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2344 2024-11-10T01:24:08,913 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:08,916 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:08,917 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201848917 2024-11-10T01:24:08,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201848917 2024-11-10T01:24:08,926 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:08,929 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201848917, startPosition=0, beingWritten=true 2024-11-10T01:24:08,942 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_0_ isCompressionEnabled=false]/testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201848917, startPosition=203, beingWritten=true 2024-11-10T01:24:08,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741844_1020 (size=447) 2024-11-10T01:24:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741844_1020 (size=447) 2024-11-10T01:24:08,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741844_1020 (size=447) 2024-11-10T01:24:08,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:08,958 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201848917) 2024-11-10T01:24:08,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[0: isCompressionEnabled=false] Thread=233 (was 228) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2342 (was 2344) 2024-11-10T01:24:08,975 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=233, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2341 2024-11-10T01:24:08,977 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:08,981 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:08,982 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731201848981 2024-11-10T01:24:08,991 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731201848981 2024-11-10T01:24:08,992 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:08,996 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731201848981, startPosition=0, beingWritten=true 2024-11-10T01:24:09,005 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:09,006 DEBUG [Thread-357 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731201848981, startPosition=0, beingWritten=true 2024-11-10T01:24:09,013 DEBUG [Thread-357 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:09,014 DEBUG [Thread-357 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731201848981 to pos 460, reset compression=false 2024-11-10T01:24:09,020 DEBUG [Thread-357 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-10T01:24:09,021 DEBUG [Thread-357 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731201848981 to pos 580, reset compression=false 2024-11-10T01:24:09,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741845_1021 (size=588) 2024-11-10T01:24:09,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741845_1021 (size=588) 2024-11-10T01:24:09,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741845_1021 (size=588) 2024-11-10T01:24:09,027 DEBUG [Thread-357 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_0_ isCompressionEnabled=false]/testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D.1731201848981 2024-11-10T01:24:09,028 DEBUG [Thread-357 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:09,028 DEBUG [Thread-357 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:09,028 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:09,028 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201848981) 2024-11-10T01:24:09,036 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[0: isCompressionEnabled=false] Thread=239 (was 233) Potentially hanging thread: Thread-357 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2337 (was 2341) 2024-11-10T01:24:09,044 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=239, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2337 2024-11-10T01:24:09,045 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:09,049 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:09,050 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 2024-11-10T01:24:09,058 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 2024-11-10T01:24:09,059 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:09,060 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849060 2024-11-10T01:24:09,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,071 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849060 2024-11-10T01:24:09,073 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:09,073 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 is not closed yet, will try archiving it next time 2024-11-10T01:24:09,074 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:09,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741846_1022 (size=93) 2024-11-10T01:24:09,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741846_1022 (size=93) 2024-11-10T01:24:09,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741846_1022 (size=93) 2024-11-10T01:24:09,078 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 2024-11-10T01:24:09,177 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 2024-11-10T01:24:09,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741848_1024 (size=85) 2024-11-10T01:24:09,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741848_1024 (size=85) 2024-11-10T01:24:09,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741848_1024 (size=85) 2024-11-10T01:24:09,195 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050, startPosition=0, beingWritten=false 2024-11-10T01:24:09,197 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 2024-11-10T01:24:09,201 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 for the last time to pos 0, reset compression=false 2024-11-10T01:24:09,202 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 2024-11-10T01:24:09,207 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 2024-11-10T01:24:09,207 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 and length of the file is 85 2024-11-10T01:24:09,208 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849050 2024-11-10T01:24:09,208 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_0_ isCompressionEnabled=false]/testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D.1731201849060, startPosition=0, beingWritten=true 2024-11-10T01:24:09,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741847_1023 (size=218) 2024-11-10T01:24:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741847_1023 (size=218) 2024-11-10T01:24:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741847_1023 (size=218) 2024-11-10T01:24:09,220 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:09,220 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201849060) 2024-11-10T01:24:09,228 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[0: isCompressionEnabled=false] Thread=244 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2327 (was 2337) 2024-11-10T01:24:09,236 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=244, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2327 2024-11-10T01:24:09,237 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:09,241 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:09,242 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242 2024-11-10T01:24:09,250 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242 2024-11-10T01:24:09,254 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:09,257 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242, startPosition=0, beingWritten=true 2024-11-10T01:24:09,265 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:09,266 DEBUG [Thread-395 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242, startPosition=0, beingWritten=true 2024-11-10T01:24:09,275 WARN [Thread-395 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:09,296 DEBUG [Thread-395 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242, startPosition=0, beingWritten=true 2024-11-10T01:24:09,304 WARN [Thread-395 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:09,335 DEBUG [Thread-395 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242, startPosition=0, beingWritten=true 2024-11-10T01:24:09,345 WARN [Thread-395 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:09,386 DEBUG [Thread-395 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242, startPosition=0, beingWritten=true 2024-11-10T01:24:09,398 WARN [Thread-395 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:09,449 DEBUG [Thread-395 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242, startPosition=0, beingWritten=true 2024-11-10T01:24:09,461 WARN [Thread-395 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$FailingWALEntryFilter.filter(TestBasicWALEntryStream.java:594) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:09,522 DEBUG [Thread-395 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242, startPosition=0, beingWritten=true 2024-11-10T01:24:09,532 DEBUG [Thread-395 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:09,532 DEBUG [Thread-395 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242 to pos 460, reset compression=false 2024-11-10T01:24:09,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741849_1025 (size=468) 2024-11-10T01:24:09,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741849_1025 (size=468) 2024-11-10T01:24:09,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741849_1025 (size=468) 2024-11-10T01:24:09,542 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:09,542 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201849242) 2024-11-10T01:24:09,545 DEBUG [Thread-395 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_0_ isCompressionEnabled=false]/testReplicationSourceWALReaderWithFailingFilter_0_+isCompressionEnabled%3Dfalse%5D.1731201849242 2024-11-10T01:24:09,546 DEBUG [Thread-395 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:09,546 DEBUG [Thread-395 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:09,552 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[0: isCompressionEnabled=false] Thread=250 (was 244) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-395 app//org.apache.logging.log4j.core.impl.Log4jLogEvent.(Log4jLogEvent.java:465) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.(Log4jLogEvent.java:454) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.(Log4jLogEvent.java:355) app//org.apache.logging.log4j.core.impl.DefaultLogEventFactory.createEvent(DefaultLogEventFactory.java:74) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:550) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2034) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1899) app//org.apache.logging.slf4j.Log4jLogger.debug(Log4jLogger.java:124) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.dequeueCurrentLog(WALEntryStream.java:456) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:374) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2316 (was 2327) 2024-11-10T01:24:09,560 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=250, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2316 2024-11-10T01:24:09,561 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:09,565 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEmptyStream_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:09,566 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731201849566 2024-11-10T01:24:09,574 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731201849566 2024-11-10T01:24:09,580 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:09,584 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEmptyStream_0_ isCompressionEnabled=false]/testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D.1731201849566, startPosition=0, beingWritten=true 2024-11-10T01:24:09,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741850_1026 (size=93) 2024-11-10T01:24:09,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741850_1026 (size=93) 2024-11-10T01:24:09,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741850_1026 (size=93) 2024-11-10T01:24:09,602 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:09,602 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201849566) 2024-11-10T01:24:09,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[0: isCompressionEnabled=false] Thread=255 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2310 (was 2316) 2024-11-10T01:24:09,619 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=255, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2308 2024-11-10T01:24:09,620 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:09,624 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:09,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849624 2024-11-10T01:24:09,633 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849624 2024-11-10T01:24:09,639 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:09,642 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849624, startPosition=0, beingWritten=true 2024-11-10T01:24:09,651 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:09,652 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849624, startPosition=210, beingWritten=true 2024-11-10T01:24:09,659 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849659 2024-11-10T01:24:09,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,668 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849624 with entries=3, filesize=460 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849659 2024-11-10T01:24:09,672 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:09,672 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849624 is not closed yet, will try archiving it next time 2024-11-10T01:24:09,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741851_1027 (size=468) 2024-11-10T01:24:09,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741851_1027 (size=468) 2024-11-10T01:24:09,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741851_1027 (size=468) 2024-11-10T01:24:09,676 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:09,676 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849624, startPosition=335, beingWritten=false 2024-11-10T01:24:09,680 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:09,680 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849624 2024-11-10T01:24:09,780 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_0_ isCompressionEnabled=false]/testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D.1731201849659, startPosition=0, beingWritten=true 2024-11-10T01:24:09,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741852_1028 (size=218) 2024-11-10T01:24:09,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741852_1028 (size=218) 2024-11-10T01:24:09,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741852_1028 (size=218) 2024-11-10T01:24:09,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:09,797 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201849659) 2024-11-10T01:24:09,806 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[0: isCompressionEnabled=false] Thread=260 (was 255) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2293 (was 2308) 2024-11-10T01:24:09,814 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=260, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2290 2024-11-10T01:24:09,816 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:09,819 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:09,820 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201849820 2024-11-10T01:24:09,827 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201849820 2024-11-10T01:24:09,830 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:09,835 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201849820, startPosition=0, beingWritten=true 2024-11-10T01:24:09,844 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_0_ isCompressionEnabled=false]/testPosition_0_+isCompressionEnabled%3Dfalse%5D.1731201849820, startPosition=210, beingWritten=true 2024-11-10T01:24:09,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741853_1029 (size=468) 2024-11-10T01:24:09,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741853_1029 (size=468) 2024-11-10T01:24:09,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741853_1029 (size=468) 2024-11-10T01:24:09,863 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:09,863 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201849820) 2024-11-10T01:24:09,872 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[0: isCompressionEnabled=false] Thread=265 (was 260) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #35] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 567), ProcessCount=11 (was 11), AvailableMemoryMB=2286 (was 2290) 2024-11-10T01:24:09,884 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=265, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=2285 2024-11-10T01:24:09,885 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:09,888 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:09,889 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201849889 2024-11-10T01:24:09,900 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201849889 2024-11-10T01:24:09,909 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:09,913 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201849913 2024-11-10T01:24:09,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,929 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201849889 with entries=1, filesize=210 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201849913 2024-11-10T01:24:09,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:09,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201849889 is not closed yet, will try archiving it next time 2024-11-10T01:24:09,933 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:09,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741854_1030 (size=218) 2024-11-10T01:24:09,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741854_1030 (size=218) 2024-11-10T01:24:09,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741854_1030 (size=218) 2024-11-10T01:24:09,937 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201849889, startPosition=0, beingWritten=false 2024-11-10T01:24:09,941 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_0_ isCompressionEnabled=false]/testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D.1731201849889 2024-11-10T01:24:09,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:09,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741855_1031 (size=93) 2024-11-10T01:24:09,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741855_1031 (size=93) 2024-11-10T01:24:09,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741855_1031 (size=93) 2024-11-10T01:24:09,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:09,954 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_0_+isCompressionEnabled%3Dfalse%5D:(num 1731201849913) 2024-11-10T01:24:09,963 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[0: isCompressionEnabled=false] Thread=270 (was 265) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #48] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 567) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2278 (was 2285) 2024-11-10T01:24:09,973 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=270, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2278 2024-11-10T01:24:09,975 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:09,978 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:09,979 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201849979 2024-11-10T01:24:09,990 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201849979 2024-11-10T01:24:09,993 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:09,998 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201849979, startPosition=0, beingWritten=true 2024-11-10T01:24:10,006 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201849979: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:10,011 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testNewEntriesWhileStreaming_1_ isCompressionEnabled=true]/testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201849979 to pos 197, reset compression=false 2024-11-10T01:24:10,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741856_1032 (size=317) 2024-11-10T01:24:10,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741856_1032 (size=317) 2024-11-10T01:24:10,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741856_1032 (size=317) 2024-11-10T01:24:10,025 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:10,025 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testNewEntriesWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201849979) 2024-11-10T01:24:10,033 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testNewEntriesWhileStreaming[1: isCompressionEnabled=true] Thread=274 (was 270) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2270 (was 2278) 2024-11-10T01:24:10,042 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=274, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2270 2024-11-10T01:24:10,043 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:10,047 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:10,048 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047 2024-11-10T01:24:10,056 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047 2024-11-10T01:24:10,059 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:10,064 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047, startPosition=0, beingWritten=true 2024-11-10T01:24:10,070 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:10,074 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:10,077 DEBUG [Thread-476 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047, startPosition=0, beingWritten=true 2024-11-10T01:24:10,086 DEBUG [Thread-476 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:10,090 WARN [Thread-476 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:10,111 DEBUG [Thread-476 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047, startPosition=0, beingWritten=true 2024-11-10T01:24:10,117 DEBUG [Thread-476 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:10,120 WARN [Thread-476 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:10,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741834_1010 (size=218) 2024-11-10T01:24:10,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741833_1009 (size=93) 2024-11-10T01:24:10,151 DEBUG [Thread-476 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047, startPosition=0, beingWritten=true 2024-11-10T01:24:10,156 DEBUG [Thread-476 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:10,159 WARN [Thread-476 {}] regionserver.ReplicationSourceWALReader(194): Failed to filter WAL entries and the filter let us retry later org.apache.hadoop.hbase.replication.regionserver.WALEntryFilterRetryableException: failing filter at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream$PartialWALEntryFailingWALEntryFilter.filter(TestBasicWALEntryStream.java:906) ~[test-classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.filterEntry(ReplicationSourceWALReader.java:290) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.readWALEntries(ReplicationSourceWALReader.java:239) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:173) ~[classes/:?] 2024-11-10T01:24:10,200 DEBUG [Thread-476 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047, startPosition=0, beingWritten=true 2024-11-10T01:24:10,209 DEBUG [Thread-476 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:10,212 DEBUG [Thread-476 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:10,212 DEBUG [Thread-476 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047 to pos 314, reset compression=false 2024-11-10T01:24:10,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741857_1033 (size=322) 2024-11-10T01:24:10,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741857_1033 (size=322) 2024-11-10T01:24:10,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741857_1033 (size=322) 2024-11-10T01:24:10,230 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:10,230 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201850047) 2024-11-10T01:24:10,239 DEBUG [Thread-476 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047 to pos 314, reset compression=false 2024-11-10T01:24:10,241 INFO [Thread-476 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047 2024-11-10T01:24:10,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithPartialWALEntryFailingFilter[1: isCompressionEnabled=true] Thread=283 (was 274) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #58] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-476 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@4958c6a0[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@5e7c267e[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2261 (was 2270) 2024-11-10T01:24:10,247 DEBUG [Thread-476 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithPartialWALEntryFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201850047 2024-11-10T01:24:10,247 DEBUG [Thread-476 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:10,247 DEBUG [Thread-476 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:10,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=283, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2260 2024-11-10T01:24:10,252 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:10,256 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:10,256 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1731201850256 2024-11-10T01:24:10,264 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D.1731201850256 2024-11-10T01:24:10,268 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:10,276 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor log.1.true 2024-11-10T01:24:10,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741859_1035 (size=225) 2024-11-10T01:24:10,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741859_1035 (size=225) 2024-11-10T01:24:10,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741859_1035 (size=225) 2024-11-10T01:24:10,288 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-10T01:24:10,290 DEBUG [Thread-504 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/user/jenkins/log.1.true, startPosition=0, beingWritten=false 2024-11-10T01:24:10,293 DEBUG [Thread-504 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/user/jenkins/log.1.true: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:10,295 DEBUG [Thread-504 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/user/jenkins/log.1.true 2024-11-10T01:24:10,295 DEBUG [Thread-504 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:10,296 DEBUG [Thread-504 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/user/jenkins/log.2.true, startPosition=0, beingWritten=false 2024-11-10T01:24:10,297 WARN [Thread-504 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:44367/user/jenkins/log.2.true, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-10T01:24:10,297 DEBUG [Thread-504 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/user/jenkins/log.2.true 2024-11-10T01:24:10,297 DEBUG [Thread-504 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:10,297 DEBUG [Thread-504 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:10,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741858_1034 (size=93) 2024-11-10T01:24:10,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741858_1034 (size=93) 2024-11-10T01:24:10,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741858_1034 (size=93) 2024-11-10T01:24:10,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741835_1011 (size=2593) 2024-11-10T01:24:10,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741836_1012 (size=1343) 2024-11-10T01:24:10,487 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T01:24:10,711 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:10,711 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueueWithMultipleLogs_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201850256) 2024-11-10T01:24:10,726 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueueWithMultipleLogs[1: isCompressionEnabled=true] Thread=291 (was 283) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@138080ed[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #60] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@1d476ced[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2248 (was 2260) 2024-11-10T01:24:10,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=291, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2248 2024-11-10T01:24:10,737 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:10,742 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:10,743 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 2024-11-10T01:24:10,752 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 2024-11-10T01:24:10,754 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:10,756 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 2024-11-10T01:24:10,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,768 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 2024-11-10T01:24:10,770 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:10,770 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 is not closed yet, will try archiving it next time 2024-11-10T01:24:10,771 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:10,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741860_1036 (size=93) 2024-11-10T01:24:10,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741860_1036 (size=93) 2024-11-10T01:24:10,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741860_1036 (size=93) 2024-11-10T01:24:10,775 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 2024-11-10T01:24:10,872 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 2024-11-10T01:24:10,877 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:10,877 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:10,878 DEBUG [Thread-526 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743, startPosition=0, beingWritten=false 2024-11-10T01:24:10,880 INFO [Thread-526 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 2024-11-10T01:24:10,881 WARN [Thread-526 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-10T01:24:10,882 DEBUG [Thread-526 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850743 2024-11-10T01:24:10,882 DEBUG [Thread-526 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755, startPosition=0, beingWritten=true 2024-11-10T01:24:10,887 DEBUG [Thread-526 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:10,898 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 to pos 0, reset compression=false 2024-11-10T01:24:10,915 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 to pos 0, reset compression=false 2024-11-10T01:24:10,933 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 to pos 0, reset compression=false 2024-11-10T01:24:10,953 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 to pos 0, reset compression=false 2024-11-10T01:24:10,973 DEBUG [Thread-526 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 to pos 0, reset compression=false 2024-11-10T01:24:10,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:10,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741861_1037 (size=93) 2024-11-10T01:24:10,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741861_1037 (size=93) 2024-11-10T01:24:10,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741861_1037 (size=93) 2024-11-10T01:24:10,984 DEBUG [Thread-526 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 for the last time to pos 0, reset compression=false 2024-11-10T01:24:10,987 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:10,987 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201850755) 2024-11-10T01:24:10,993 INFO [Thread-526 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 2024-11-10T01:24:10,994 DEBUG [Thread-526 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 and length of the file is 93 2024-11-10T01:24:10,994 DEBUG [Thread-526 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionInOldWALsDirectory_1_ isCompressionEnabled=true]/testEOFExceptionInOldWALsDirectory_1_+isCompressionEnabled%3Dtrue%5D.1731201850755 2024-11-10T01:24:10,994 DEBUG [Thread-526 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:10,994 DEBUG [Thread-526 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:10,999 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionInOldWALsDirectory[1: isCompressionEnabled=true] Thread=297 (was 291) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #70] java.base@17.0.11/sun.nio.ch.SocketDispatcher.read0(Native Method) java.base@17.0.11/sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:47) java.base@17.0.11/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:330) java.base@17.0.11/sun.nio.ch.IOUtil.read(IOUtil.java:296) java.base@17.0.11/sun.nio.ch.IOUtil.read(IOUtil.java:259) java.base@17.0.11/sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:417) app//org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:628) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-526 app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$ClientReadStatusProto.access$28602(DataTransferProtos.java:27628) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$ClientReadStatusProto$Builder.buildPartial0(DataTransferProtos.java:27921) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$ClientReadStatusProto$Builder.buildPartial(DataTransferProtos.java:27912) app//org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$ClientReadStatusProto$Builder.build(DataTransferProtos.java:27902) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.writeReadResult(BlockReaderRemote.java:346) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.sendReadResult(BlockReaderRemote.java:329) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.readNextPacket(BlockReaderRemote.java:235) app//org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.read(BlockReaderRemote.java:146) app//org.apache.hadoop.hdfs.ByteArrayStrategy.readFromBlock(ReaderStrategy.java:118) app//org.apache.hadoop.hdfs.DFSInputStream.readBuffer(DFSInputStream.java:789) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:855) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.DataInputStream.readInt(DataInputStream.java:381) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:212) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2239 (was 2248) 2024-11-10T01:24:11,007 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=297, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2239 2024-11-10T01:24:11,007 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:11,011 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:11,012 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851012 2024-11-10T01:24:11,020 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851012 2024-11-10T01:24:11,023 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:11,026 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851026 2024-11-10T01:24:11,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,036 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851012 with entries=10, filesize=707 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851026 2024-11-10T01:24:11,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:11,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851012 is not closed yet, will try archiving it next time 2024-11-10T01:24:11,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741862_1038 (size=715) 2024-11-10T01:24:11,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741862_1038 (size=715) 2024-11-10T01:24:11,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741862_1038 (size=715) 2024-11-10T01:24:11,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741863_1039 (size=440) 2024-11-10T01:24:11,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741863_1039 (size=440) 2024-11-10T01:24:11,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741863_1039 (size=440) 2024-11-10T01:24:11,046 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=10, replicationBatchQueueCapacity=1 2024-11-10T01:24:11,046 DEBUG [Thread-549 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851012, startPosition=0, beingWritten=false 2024-11-10T01:24:11,049 DEBUG [Thread-549 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851012: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,052 DEBUG [Thread-549 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-10T01:24:11,052 DEBUG [Thread-549 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851012 2024-11-10T01:24:11,052 DEBUG [Thread-549 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851026, startPosition=0, beingWritten=false 2024-11-10T01:24:11,055 DEBUG [Thread-549 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851026: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,057 DEBUG [Thread-549 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderRecovered_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D.1731201851026 2024-11-10T01:24:11,057 DEBUG [Thread-549 {}] regionserver.ReplicationSourceWALReader(177): Read 5 WAL entries eligible for replication 2024-11-10T01:24:11,057 DEBUG [Thread-549 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:11,057 DEBUG [Thread-549 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:11,061 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:11,061 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderRecovered_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201851026) 2024-11-10T01:24:11,069 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderRecovered[1: isCompressionEnabled=true] Thread=302 (was 297) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #72] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2232 (was 2239) 2024-11-10T01:24:11,077 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=302, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2232 2024-11-10T01:24:11,078 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:11,081 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:11,082 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201851082 2024-11-10T01:24:11,091 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEOFExceptionForRecoveredQueue_1_ isCompressionEnabled=true]/testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201851082 2024-11-10T01:24:11,092 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:11,099 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=10 2024-11-10T01:24:11,100 DEBUG [Thread-559 {}] regionserver.WALEntryStream(254): Creating new reader emptyLog, startPosition=0, beingWritten=false 2024-11-10T01:24:11,101 WARN [Thread-559 {}] regionserver.WALEntryStream(269): EOF while trying to open WAL reader for path: emptyLog, startPosition=0 org.apache.hadoop.hbase.regionserver.wal.WALHeaderEOFException: EOF while reading PB WAL magic at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:221) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.init(AbstractProtobufWALReader.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createTailingReader(WALFactory.java:521) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.prepareReader(WALEntryStream.java:257) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:332) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:203) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readHeader(AbstractProtobufWALReader.java:219) ~[classes/:?] ... 6 more 2024-11-10T01:24:11,102 DEBUG [Thread-559 {}] regionserver.WALEntryStream(456): EOF, closing emptyLog 2024-11-10T01:24:11,102 DEBUG [Thread-559 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:11,102 DEBUG [Thread-559 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:11,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741864_1040 (size=93) 2024-11-10T01:24:11,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741864_1040 (size=93) 2024-11-10T01:24:11,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741864_1040 (size=93) 2024-11-10T01:24:11,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:11,109 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEOFExceptionForRecoveredQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201851082) 2024-11-10T01:24:11,117 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEOFExceptionForRecoveredQueue[1: isCompressionEnabled=true] Thread=307 (was 302) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2228 (was 2232) 2024-11-10T01:24:11,125 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=307, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2228 2024-11-10T01:24:11,126 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:11,129 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:11,130 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851129 2024-11-10T01:24:11,138 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851129 2024-11-10T01:24:11,140 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:11,147 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146 2024-11-10T01:24:11,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,156 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851129 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146 2024-11-10T01:24:11,157 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:11,157 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851129 is not closed yet, will try archiving it next time 2024-11-10T01:24:11,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741865_1041 (size=212) 2024-11-10T01:24:11,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741865_1041 (size=212) 2024-11-10T01:24:11,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741865_1041 (size=212) 2024-11-10T01:24:11,162 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:11,164 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:11,165 DEBUG [Thread-576 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851129, startPosition=0, beingWritten=false 2024-11-10T01:24:11,168 DEBUG [Thread-576 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851129: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,169 DEBUG [Thread-576 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851129 2024-11-10T01:24:11,169 DEBUG [Thread-576 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-10T01:24:11,169 DEBUG [Thread-576 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146, startPosition=0, beingWritten=true 2024-11-10T01:24:11,178 DEBUG [Thread-576 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,186 DEBUG [Thread-576 {}] regionserver.ReplicationSourceWALReader(177): Read 20 WAL entries eligible for replication 2024-11-10T01:24:11,186 DEBUG [Thread-576 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146 to pos 1273, reset compression=false 2024-11-10T01:24:11,186 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 2024-11-10T01:24:11,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,198 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146 with entries=20, filesize=1.24 KB; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 2024-11-10T01:24:11,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:11,199 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146 is not closed yet, will try archiving it next time 2024-11-10T01:24:11,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741866_1042 (size=1281) 2024-11-10T01:24:11,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741866_1042 (size=1281) 2024-11-10T01:24:11,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741866_1042 (size=1281) 2024-11-10T01:24:11,219 DEBUG [Thread-576 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146 to pos 1273, reset compression=false 2024-11-10T01:24:11,222 DEBUG [Thread-576 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851146 2024-11-10T01:24:11,223 DEBUG [Thread-576 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186, startPosition=0, beingWritten=true 2024-11-10T01:24:11,229 DEBUG [Thread-576 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,235 DEBUG [Thread-576 {}] regionserver.ReplicationSourceWALReader(177): Read 10 WAL entries eligible for replication 2024-11-10T01:24:11,236 DEBUG [Thread-576 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 to pos 707, reset compression=false 2024-11-10T01:24:11,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741867_1043 (size=715) 2024-11-10T01:24:11,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741867_1043 (size=715) 2024-11-10T01:24:11,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741867_1043 (size=715) 2024-11-10T01:24:11,249 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:11,249 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201851186) 2024-11-10T01:24:11,257 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWrongPosition[1: isCompressionEnabled=true] Thread=313 (was 307) Potentially hanging thread: Thread-576 java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method) java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017) java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method) java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017) java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.hbase.thirdparty.com.google.protobuf.Descriptors$FileDescriptor.internalBuildGeneratedFileFrom(Descriptors.java:466) app//org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.(HBaseProtos.java:27113) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.(WALProtos.java:15202) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.internalGetFieldAccessorTable(WALProtos.java:1640) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.getDescriptorForType(GeneratedMessage.java:106) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:151) app//org.apache.hbase.thirdparty.com.google.protobuf.MessageReflection.findMissingFields(MessageReflection.java:185) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.newUninitializedMessageException(AbstractMessage.java:446) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage.newUninitializedMessageException(AbstractMessage.java:284) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.newUninitializedMessageException(AbstractParser.java:30) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:44) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) app//org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) app//org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) app//org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #75] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2215 (was 2228) 2024-11-10T01:24:11,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=313, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2213 2024-11-10T01:24:11,267 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:11,271 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALKeySerialization_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:11,271 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1731201851271 2024-11-10T01:24:11,281 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALKeySerialization_1_ isCompressionEnabled=true]/testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D.1731201851271 2024-11-10T01:24:11,288 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:11,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741868_1044 (size=93) 2024-11-10T01:24:11,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741868_1044 (size=93) 2024-11-10T01:24:11,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741868_1044 (size=93) 2024-11-10T01:24:11,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:11,304 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALKeySerialization_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201851271) 2024-11-10T01:24:11,314 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALKeySerialization[1: isCompressionEnabled=true] Thread=318 (was 313) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2205 (was 2213) 2024-11-10T01:24:11,323 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=318, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2204 2024-11-10T01:24:11,324 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:11,326 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:11,327 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731201851327 2024-11-10T01:24:11,335 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731201851327 2024-11-10T01:24:11,342 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:11,345 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731201851327, startPosition=0, beingWritten=true 2024-11-10T01:24:11,354 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731201851327: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,355 INFO [Thread-576 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=707, currentPosition=715 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-10T01:24:11,356 DEBUG [Thread-576 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 for the last time to pos 707, reset compression=false 2024-11-10T01:24:11,357 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:11,358 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:11,358 INFO [Thread-576 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 2024-11-10T01:24:11,362 INFO [Thread-576 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 2024-11-10T01:24:11,362 DEBUG [Thread-576 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 and length of the file is 715 2024-11-10T01:24:11,363 DEBUG [Thread-576 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWrongPosition_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWrongPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201851186 2024-11-10T01:24:11,363 DEBUG [Thread-576 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:11,363 DEBUG [Thread-576 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:11,460 DEBUG [Thread-606 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731201851327, startPosition=0, beingWritten=true 2024-11-10T01:24:11,465 DEBUG [Thread-606 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731201851327: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,467 DEBUG [Thread-606 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:11,468 DEBUG [Thread-606 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731201851327 to pos 322, reset compression=false 2024-11-10T01:24:11,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741869_1045 (size=330) 2024-11-10T01:24:11,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741869_1045 (size=330) 2024-11-10T01:24:11,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741869_1045 (size=330) 2024-11-10T01:24:11,475 DEBUG [Thread-606 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderDisabled_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D.1731201851327 2024-11-10T01:24:11,475 DEBUG [Thread-606 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:11,475 DEBUG [Thread-606 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:11,476 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:11,476 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderDisabled_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201851327) 2024-11-10T01:24:11,485 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderDisabled[1: isCompressionEnabled=true] Thread=325 (was 318) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #78] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-606 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:343) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.replicationDone(ReplicationSourceWALReader.java:123) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:151) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2193 (was 2204) 2024-11-10T01:24:11,494 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=325, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2193 2024-11-10T01:24:11,494 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:11,497 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:11,498 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851498 2024-11-10T01:24:11,507 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851498 2024-11-10T01:24:11,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:11,512 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:11,512 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851498, startPosition=0, beingWritten=true 2024-11-10T01:24:11,517 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851498: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,520 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851520 2024-11-10T01:24:11,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,528 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851498 with entries=3, filesize=309 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851520 2024-11-10T01:24:11,529 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:11,529 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851498 is not closed yet, will try archiving it next time 2024-11-10T01:24:11,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741870_1046 (size=317) 2024-11-10T01:24:11,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741870_1046 (size=317) 2024-11-10T01:24:11,532 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:11,532 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:11,532 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851498 for the last time to pos 253, reset compression=false 2024-11-10T01:24:11,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741870_1046 (size=317) 2024-11-10T01:24:11,541 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:11,541 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851498 2024-11-10T01:24:11,642 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851520, startPosition=0, beingWritten=true 2024-11-10T01:24:11,650 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testLogRollWhileStreaming_1_ isCompressionEnabled=true]/testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D.1731201851520: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741871_1047 (size=205) 2024-11-10T01:24:11,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741871_1047 (size=205) 2024-11-10T01:24:11,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741871_1047 (size=205) 2024-11-10T01:24:11,662 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:11,662 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testLogRollWhileStreaming_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201851520) 2024-11-10T01:24:11,673 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testLogRollWhileStreaming[1: isCompressionEnabled=true] Thread=330 (was 325) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #82] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2184 (was 2193) 2024-11-10T01:24:11,681 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=330, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2184 2024-11-10T01:24:11,681 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:11,685 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:11,686 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851686 2024-11-10T01:24:11,694 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851686 2024-11-10T01:24:11,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:11,700 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:11,700 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851686, startPosition=0, beingWritten=true 2024-11-10T01:24:11,705 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851686: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,706 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851706 2024-11-10T01:24:11,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,715 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851686 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851706 2024-11-10T01:24:11,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741872_1048 (size=212) 2024-11-10T01:24:11,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741872_1048 (size=212) 2024-11-10T01:24:11,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741872_1048 (size=212) 2024-11-10T01:24:11,725 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:11,726 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:11,727 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851686 for the last time to pos 204, reset compression=false 2024-11-10T01:24:11,730 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851686 and length of the file is 212 2024-11-10T01:24:11,730 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851686 2024-11-10T01:24:11,830 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851706, startPosition=0, beingWritten=true 2024-11-10T01:24:11,836 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testCleanClosedWALs_1_ isCompressionEnabled=true]/testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D.1731201851706: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:11,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741873_1049 (size=212) 2024-11-10T01:24:11,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741873_1049 (size=212) 2024-11-10T01:24:11,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741873_1049 (size=212) 2024-11-10T01:24:11,845 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:11,845 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testCleanClosedWALs_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201851706) 2024-11-10T01:24:11,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testCleanClosedWALs[1: isCompressionEnabled=true] Thread=335 (was 330) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #49] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2176 (was 2184) 2024-11-10T01:24:11,866 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=335, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2175 2024-11-10T01:24:11,867 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:11,870 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:11,871 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731201851871 2024-11-10T01:24:11,879 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731201851871 2024-11-10T01:24:11,885 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:11,889 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731201851871, startPosition=0, beingWritten=true 2024-11-10T01:24:11,895 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731201851871: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:11,896 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:612) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-10T01:24:12,897 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731201851871 to pos 197, reset compression=true 2024-11-10T01:24:12,904 WARN [Time-limited test {}] wal.ProtobufWALTailingReader(207): EOF while reading 1 WAL KVs; started reading at 225 and read up to 252 java.io.EOFException: Partial cell read at org.apache.hadoop.hbase.codec.BaseDecoder.rethrowEofException(BaseDecoder.java:86) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:70) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALEdit.readFromCells(WALEdit.java:326) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALEdit(ProtobufWALTailingReader.java:201) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:260) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.TestBasicWALEntryStream.testReadBeyondCommittedLength(TestBasicWALEntryStream.java:615) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.WALCellCodec$CompressedKvDecoder.parseCell(WALCellCodec.java:340) ~[classes/:?] at org.apache.hadoop.hbase.codec.BaseDecoder.advance(BaseDecoder.java:66) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 42 more 2024-11-10T01:24:12,905 DEBUG [Time-limited test {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReadBeyondCommittedLength_1_ isCompressionEnabled=true]/testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D.1731201851871 to pos 197, reset compression=true 2024-11-10T01:24:12,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741874_1050 (size=261) 2024-11-10T01:24:12,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741874_1050 (size=261) 2024-11-10T01:24:12,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741874_1050 (size=261) 2024-11-10T01:24:12,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:12,923 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReadBeyondCommittedLength_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201851871) 2024-11-10T01:24:12,932 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReadBeyondCommittedLength[1: isCompressionEnabled=true] Thread=340 (was 335) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #88] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2162 (was 2175) 2024-11-10T01:24:12,943 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=340, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2160 2024-11-10T01:24:12,943 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:12,947 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:12,947 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201852947 2024-11-10T01:24:12,953 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201852947 2024-11-10T01:24:12,956 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:12,961 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201852947, startPosition=0, beingWritten=true 2024-11-10T01:24:12,966 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201852947: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:12,970 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201852947, startPosition=197, beingWritten=true 2024-11-10T01:24:12,975 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testResumeStreamingFromPosition_1_ isCompressionEnabled=true]/testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201852947: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:12,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:12,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741875_1051 (size=317) 2024-11-10T01:24:12,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741875_1051 (size=317) 2024-11-10T01:24:12,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741875_1051 (size=317) 2024-11-10T01:24:12,983 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:12,983 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testResumeStreamingFromPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201852947) 2024-11-10T01:24:12,992 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testResumeStreamingFromPosition[1: isCompressionEnabled=true] Thread=345 (was 340) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #53] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2155 (was 2160) 2024-11-10T01:24:13,001 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=345, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2155 2024-11-10T01:24:13,002 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:13,005 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:13,006 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 2024-11-10T01:24:13,013 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 2024-11-10T01:24:13,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:13,022 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006, startPosition=0, beingWritten=true 2024-11-10T01:24:13,029 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,032 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:13,033 DEBUG [Thread-676 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006, startPosition=0, beingWritten=true 2024-11-10T01:24:13,037 DEBUG [Thread-676 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,039 DEBUG [Thread-676 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:13,039 DEBUG [Thread-676 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 to pos 314, reset compression=false 2024-11-10T01:24:13,045 DEBUG [Thread-676 {}] regionserver.ReplicationSourceWALReader(177): Read 1 WAL entries eligible for replication 2024-11-10T01:24:13,045 DEBUG [Thread-676 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 to pos 372, reset compression=false 2024-11-10T01:24:13,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741876_1052 (size=380) 2024-11-10T01:24:13,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741876_1052 (size=380) 2024-11-10T01:24:13,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741876_1052 (size=380) 2024-11-10T01:24:13,052 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:13,052 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201853006) 2024-11-10T01:24:13,052 INFO [Thread-676 {}] wal.ProtobufWALTailingReader(152): Reached partial WAL Trailer(EOF) while reading WALKey, originalPosition=372, currentPosition=380 org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: encoded_region_name, table_name, log_sequence_number, write_time at org.apache.hbase.thirdparty.com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:45) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:209) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:214) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:25) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessage.parseWithIOException(GeneratedMessage.java:324) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos$WALKey.parseFrom(WALProtos.java:2322) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:127) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) ~[classes/:?] 2024-11-10T01:24:13,054 DEBUG [Thread-676 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 for the last time to pos 372, reset compression=false 2024-11-10T01:24:13,056 INFO [Thread-676 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 2024-11-10T01:24:13,059 INFO [Thread-676 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 2024-11-10T01:24:13,059 DEBUG [Thread-676 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 and length of the file is 380 2024-11-10T01:24:13,059 DEBUG [Thread-676 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReader_1_ isCompressionEnabled=true]/testReplicationSourceWALReader_1_+isCompressionEnabled%3Dtrue%5D.1731201853006 2024-11-10T01:24:13,060 DEBUG [Thread-676 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:13,060 DEBUG [Thread-676 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:13,061 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReader[1: isCompressionEnabled=true] Thread=351 (was 345) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #92] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-676 java.base@17.0.11/java.lang.StringLatin1.lastIndexOf(StringLatin1.java:294) java.base@17.0.11/java.lang.String.lastIndexOf(String.java:2489) java.base@17.0.11/java.lang.String.lastIndexOf(String.java:2451) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:182) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.logging.log4j.util.LoaderUtil.loadClass(LoaderUtil.java:169) app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.loadClass(ThrowableProxyHelper.java:214) app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toExtendedStackTrace(ThrowableProxyHelper.java:112) app//org.apache.logging.log4j.core.impl.ThrowableProxy.(ThrowableProxy.java:113) app//org.apache.logging.log4j.core.impl.ThrowableProxy.(ThrowableProxy.java:97) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:629) app//org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:63) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2028) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1891) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:194) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.readWALKey(ProtobufWALTailingReader.java:152) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.next(ProtobufWALTailingReader.java:256) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.readNextEntryAndRecordReaderPosition(WALEntryStream.java:489) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:337) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #55] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2145 (was 2155) 2024-11-10T01:24:13,072 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=351, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2144 2024-11-10T01:24:13,072 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:13,077 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:13,078 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 2024-11-10T01:24:13,086 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 2024-11-10T01:24:13,088 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:13,096 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853096 2024-11-10T01:24:13,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,105 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853096 2024-11-10T01:24:13,106 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:13,106 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 is not closed yet, will try archiving it next time 2024-11-10T01:24:13,106 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:13,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741877_1053 (size=93) 2024-11-10T01:24:13,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741877_1053 (size=93) 2024-11-10T01:24:13,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741877_1053 (size=93) 2024-11-10T01:24:13,109 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 2024-11-10T01:24:13,210 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 2024-11-10T01:24:13,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741879_1055 (size=85) 2024-11-10T01:24:13,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741879_1055 (size=85) 2024-11-10T01:24:13,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741879_1055 (size=85) 2024-11-10T01:24:13,224 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078, startPosition=0, beingWritten=false 2024-11-10T01:24:13,225 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 2024-11-10T01:24:13,227 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,228 DEBUG [Time-limited test {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 for the last time to pos 0, reset compression=false 2024-11-10T01:24:13,229 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 2024-11-10T01:24:13,233 INFO [Time-limited test {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 2024-11-10T01:24:13,234 DEBUG [Time-limited test {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 and length of the file is 85 2024-11-10T01:24:13,234 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853078 2024-11-10T01:24:13,234 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853096, startPosition=0, beingWritten=true 2024-11-10T01:24:13,238 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testWALEntryStreamEOFRightAfterHeader_1_ isCompressionEnabled=true]/testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D.1731201853096: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741878_1054 (size=212) 2024-11-10T01:24:13,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741878_1054 (size=212) 2024-11-10T01:24:13,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741878_1054 (size=212) 2024-11-10T01:24:13,246 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:13,246 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testWALEntryStreamEOFRightAfterHeader_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201853096) 2024-11-10T01:24:13,256 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testWALEntryStreamEOFRightAfterHeader[1: isCompressionEnabled=true] Thread=357 (was 351) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Waiting for operation #59] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #56] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2137 (was 2144) 2024-11-10T01:24:13,265 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=357, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2137 2024-11-10T01:24:13,266 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:13,269 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:13,295 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295 2024-11-10T01:24:13,330 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295 2024-11-10T01:24:13,351 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:13,355 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295, startPosition=0, beingWritten=true 2024-11-10T01:24:13,364 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,368 INFO [Time-limited test {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=null, ReplicationSourceWALReaderThread : null inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-10T01:24:13,371 DEBUG [Thread-715 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295, startPosition=0, beingWritten=true 2024-11-10T01:24:13,379 DEBUG [Thread-715 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,381 DEBUG [Thread-715 {}] regionserver.ReplicationSourceWALReader(177): Read 3 WAL entries eligible for replication 2024-11-10T01:24:13,382 DEBUG [Thread-715 {}] regionserver.WALEntryStream(222): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295 to pos 322, reset compression=false 2024-11-10T01:24:13,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,389 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Sending block BP-1442331070-172.17.0.3-1731201839759:blk_1073741880_1056] {}] datanode.BlockSender(346): The meta file length 0 is less than the expected header length 7, indicating the meta file is corrupt 2024-11-10T01:24:13,389 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Sending block BP-1442331070-172.17.0.3-1731201839759:blk_1073741880_1056] {}] datanode.DataXceiver(658): DatanodeRegistration(127.0.0.1:35505, datanodeUuid=d75c62cb-52bb-40d0-bbce-3fbb4e8edd67, infoPort=36557, infoSecurePort=0, ipcPort=41529, storageInfo=lv=-57;cid=testClusterID;nsid=979179513;c=1731201839759):Got exception while serving BP-1442331070-172.17.0.3-1731201839759:blk_1073741880_1056 to /127.0.0.1:33712 org.apache.hadoop.hdfs.server.datanode.CorruptMetaHeaderException: The meta file length 0 is less than the expected length 7 at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:350) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T01:24:13,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741880_1056 (size=330) 2024-11-10T01:24:13,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741880_1056 (size=330) 2024-11-10T01:24:13,391 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:33712 [Sending block BP-1442331070-172.17.0.3-1731201839759:blk_1073741880_1056] {}] datanode.DataXceiver(331): 127.0.0.1:35505:DataXceiver error processing READ_BLOCK operation src: /127.0.0.1:33712 dst: /127.0.0.1:35505 org.apache.hadoop.hdfs.server.datanode.CorruptMetaHeaderException: The meta file length 0 is less than the expected length 7 at org.apache.hadoop.hdfs.server.datanode.BlockSender.(BlockSender.java:350) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java:606) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java:155) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T01:24:13,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741880_1056 (size=330) 2024-11-10T01:24:13,396 DEBUG [Thread-715 {}] regionserver.WALEntryStream(293): Reset reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295 for the last time to pos 322, reset compression=false 2024-11-10T01:24:13,398 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:13,398 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201853295) 2024-11-10T01:24:13,404 INFO [Thread-715 {}] wal.AbstractFSWALProvider(535): Log hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295 was moved to hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295 2024-11-10T01:24:13,404 DEBUG [Thread-715 {}] regionserver.WALEntryStream(449): Reached the end of hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295 and length of the file is 330 2024-11-10T01:24:13,404 DEBUG [Thread-715 {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testReplicationSourceWALReaderWithFailingFilter_1_ isCompressionEnabled=true]/testReplicationSourceWALReaderWithFailingFilter_1_+isCompressionEnabled%3Dtrue%5D.1731201853295 2024-11-10T01:24:13,405 DEBUG [Thread-715 {}] regionserver.WALEntryStream(241): No more WAL files in queue 2024-11-10T01:24:13,405 DEBUG [Thread-715 {}] regionserver.ReplicationSourceWALReader(120): Stopping the replication source wal reader 2024-11-10T01:24:13,411 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testReplicationSourceWALReaderWithFailingFilter[1: isCompressionEnabled=true] Thread=363 (was 357) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #97] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Thread-715 app//org.apache.hadoop.fs.FileSystem$Statistics.incrementBytesReadByDistance(FileSystem.java:4261) app//org.apache.hadoop.hdfs.DFSClient.updateFileSystemReadStats(DFSClient.java:3093) app//org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:864) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:919) app//org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:765) java.base@17.0.11/java.io.DataInputStream.readInt(DataInputStream.java:382) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.setTrailerIfPresent(AbstractProtobufWALReader.java:402) app//org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader.readTrailer(AbstractProtobufWALReader.java:370) app//org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader.resetTo(ProtobufWALTailingReader.java:277) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.resetReader(WALEntryStream.java:209) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.lastAttempt(WALEntryStream.java:296) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.tryAdvanceEntry(WALEntryStream.java:387) app//org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.hasNext(WALEntryStream.java:129) app//org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:149) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48770 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2091 (was 2137) 2024-11-10T01:24:13,422 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=363, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2090 2024-11-10T01:24:13,423 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:13,426 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEmptyStream_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:13,426 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731201853426 2024-11-10T01:24:13,436 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731201853426 2024-11-10T01:24:13,440 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017)] 2024-11-10T01:24:13,444 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731201853426, startPosition=0, beingWritten=true 2024-11-10T01:24:13,449 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testEmptyStream_1_ isCompressionEnabled=true]/testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D.1731201853426: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741881_1057 (size=93) 2024-11-10T01:24:13,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741881_1057 (size=93) 2024-11-10T01:24:13,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741881_1057 (size=93) 2024-11-10T01:24:13,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:13,866 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testEmptyStream_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201853426) 2024-11-10T01:24:13,880 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testEmptyStream[1: isCompressionEnabled=true] Thread=368 (was 363) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #59] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2075 (was 2090) 2024-11-10T01:24:13,894 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=368, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2075 2024-11-10T01:24:13,895 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:13,898 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:13,899 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898 2024-11-10T01:24:13,909 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898 2024-11-10T01:24:13,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:13,922 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898, startPosition=0, beingWritten=true 2024-11-10T01:24:13,926 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,929 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:13,929 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898, startPosition=204, beingWritten=true 2024-11-10T01:24:13,933 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,935 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853935 2024-11-10T01:24:13,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:13,942 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898 with entries=3, filesize=330 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853935 2024-11-10T01:24:13,943 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:13,943 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898 is not closed yet, will try archiving it next time 2024-11-10T01:24:13,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741882_1058 (size=338) 2024-11-10T01:24:13,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741882_1058 (size=338) 2024-11-10T01:24:13,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741882_1058 (size=338) 2024-11-10T01:24:13,946 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:13,946 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898, startPosition=267, beingWritten=false 2024-11-10T01:24:13,948 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:13,949 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:13,949 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853898 2024-11-10T01:24:14,049 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853935, startPosition=0, beingWritten=true 2024-11-10T01:24:14,055 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testAppendsWithRolls_1_ isCompressionEnabled=true]/testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D.1731201853935: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:14,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741883_1059 (size=212) 2024-11-10T01:24:14,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741883_1059 (size=212) 2024-11-10T01:24:14,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741883_1059 (size=212) 2024-11-10T01:24:14,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:14,067 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testAppendsWithRolls_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201853935) 2024-11-10T01:24:14,076 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testAppendsWithRolls[1: isCompressionEnabled=true] Thread=373 (was 368) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #99] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48770 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #63] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2074 (was 2075) 2024-11-10T01:24:14,086 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=373, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2073 2024-11-10T01:24:14,087 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:14,090 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testPosition_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:14,090 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201854090 2024-11-10T01:24:14,098 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201854090 2024-11-10T01:24:14,098 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38203:38203),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-10T01:24:14,102 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201854090, startPosition=0, beingWritten=true 2024-11-10T01:24:14,106 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201854090: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:14,107 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201854090, startPosition=204, beingWritten=true 2024-11-10T01:24:14,112 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testPosition_1_ isCompressionEnabled=true]/testPosition_1_+isCompressionEnabled%3Dtrue%5D.1731201854090: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:14,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741884_1060 (size=322) 2024-11-10T01:24:14,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741884_1060 (size=322) 2024-11-10T01:24:14,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741884_1060 (size=322) 2024-11-10T01:24:14,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:14,129 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testPosition_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201854090) 2024-11-10T01:24:14,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testPosition[1: isCompressionEnabled=true] Thread=378 (was 373) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48416 [Waiting for operation #101] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:48770 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2066 (was 2073) 2024-11-10T01:24:14,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=378, OpenFileDescriptor=427, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2066 2024-11-10T01:24:14,152 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T01:24:14,156 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D, suffix=, logDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true], archiveDir=hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs, maxLogs=32 2024-11-10T01:24:14,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201854157 2024-11-10T01:24:14,188 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201854157 2024-11-10T01:24:14,193 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:14,197 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201854196 2024-11-10T01:24:14,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,234 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201854157 with entries=1, filesize=204 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201854196 2024-11-10T01:24:14,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741885_1061 (size=212) 2024-11-10T01:24:14,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741885_1061 (size=212) 2024-11-10T01:24:14,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741885_1061 (size=212) 2024-11-10T01:24:14,242 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39017:39017),(127.0.0.1/127.0.0.1:36557:36557),(127.0.0.1/127.0.0.1:38203:38203)] 2024-11-10T01:24:14,242 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T01:24:14,243 DEBUG [Time-limited test {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201854157, startPosition=0, beingWritten=false 2024-11-10T01:24:14,246 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201854157: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-10T01:24:14,247 DEBUG [Time-limited test {}] regionserver.WALEntryStream(456): EOF, closing hdfs://localhost:44367/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/WALs/testSizeOfLogQueue_1_ isCompressionEnabled=true]/testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D.1731201854157 2024-11-10T01:24:14,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T01:24:14,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44509 is added to blk_1073741886_1062 (size=93) 2024-11-10T01:24:14,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35505 is added to blk_1073741886_1062 (size=93) 2024-11-10T01:24:14,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46541 is added to blk_1073741886_1062 (size=93) 2024-11-10T01:24:14,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/oldWALs 2024-11-10T01:24:14,257 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog testSizeOfLogQueue_1_+isCompressionEnabled%3Dtrue%5D:(num 1731201854196) 2024-11-10T01:24:14,266 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: replication.regionserver.TestBasicWALEntryStreamFSHLog#testSizeOfLogQueue[1: isCompressionEnabled=true] Thread=383 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1989482452_22 at /127.0.0.1:38720 [Waiting for operation #65] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=427 (was 427), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=2065 (was 2066) 2024-11-10T01:24:14,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T01:24:14,333 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d954ade{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T01:24:14,336 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@76890781{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T01:24:14,336 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T01:24:14,336 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dfa9f68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T01:24:14,336 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f179af2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir/,STOPPED} 2024-11-10T01:24:14,341 WARN [BP-1442331070-172.17.0.3-1731201839759 heartbeating to localhost/127.0.0.1:44367 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T01:24:14,341 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T01:24:14,341 WARN [BP-1442331070-172.17.0.3-1731201839759 heartbeating to localhost/127.0.0.1:44367 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1442331070-172.17.0.3-1731201839759 (Datanode Uuid d75c62cb-52bb-40d0-bbce-3fbb4e8edd67) service to localhost/127.0.0.1:44367 2024-11-10T01:24:14,341 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T01:24:14,342 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data5/current/BP-1442331070-172.17.0.3-1731201839759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T01:24:14,343 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data6/current/BP-1442331070-172.17.0.3-1731201839759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T01:24:14,343 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T01:24:14,345 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@fecc8a3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T01:24:14,346 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@cbbb804{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T01:24:14,346 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T01:24:14,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45d510a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T01:24:14,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cd634bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir/,STOPPED} 2024-11-10T01:24:14,347 WARN [BP-1442331070-172.17.0.3-1731201839759 heartbeating to localhost/127.0.0.1:44367 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T01:24:14,347 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T01:24:14,347 WARN [BP-1442331070-172.17.0.3-1731201839759 heartbeating to localhost/127.0.0.1:44367 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1442331070-172.17.0.3-1731201839759 (Datanode Uuid 7e197415-7c30-4373-9882-20e2ea42017d) service to localhost/127.0.0.1:44367 2024-11-10T01:24:14,347 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T01:24:14,348 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data3/current/BP-1442331070-172.17.0.3-1731201839759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T01:24:14,348 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data4/current/BP-1442331070-172.17.0.3-1731201839759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T01:24:14,349 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T01:24:14,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5a86aa33{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T01:24:14,355 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53cf346{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T01:24:14,355 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T01:24:14,355 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d2a62e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T01:24:14,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@309d3e97{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir/,STOPPED} 2024-11-10T01:24:14,361 WARN [BP-1442331070-172.17.0.3-1731201839759 heartbeating to localhost/127.0.0.1:44367 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T01:24:14,361 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T01:24:14,361 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T01:24:14,361 WARN [BP-1442331070-172.17.0.3-1731201839759 heartbeating to localhost/127.0.0.1:44367 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1442331070-172.17.0.3-1731201839759 (Datanode Uuid d3ce9723-82d4-40c3-86ff-701dbee99167) service to localhost/127.0.0.1:44367 2024-11-10T01:24:14,362 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data1/current/BP-1442331070-172.17.0.3-1731201839759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T01:24:14,362 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/cluster_94f68fa9-3554-3de6-d109-f277eaa43145/data/data2/current/BP-1442331070-172.17.0.3-1731201839759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T01:24:14,362 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T01:24:14,372 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44567ea4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T01:24:14,373 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@216da5e5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T01:24:14,374 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T01:24:14,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a9b22e2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T01:24:14,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c2d1933{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce1fec99-c394-f7b9-e513-1c767848af92/hadoop.log.dir/,STOPPED} 2024-11-10T01:24:14,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down