2024-11-18 17:35:05,718 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@782859e 2024-11-18 17:35:05,770 main DEBUG Took 0.049057 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 17:35:05,771 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 17:35:05,771 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 17:35:05,773 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 17:35:05,774 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,807 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 17:35:05,847 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,858 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,859 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,861 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,861 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,862 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,872 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,873 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,877 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,878 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,878 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,880 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,883 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,888 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,889 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,889 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,890 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,890 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,891 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 17:35:05,891 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,892 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 17:35:05,894 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 17:35:05,896 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 17:35:05,931 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 17:35:05,932 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 17:35:05,933 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 17:35:05,936 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 17:35:05,951 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 17:35:05,955 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 17:35:05,957 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 17:35:05,958 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 17:35:05,958 main DEBUG createAppenders(={Console}) 2024-11-18 17:35:05,959 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@782859e initialized 2024-11-18 17:35:05,959 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@782859e 2024-11-18 17:35:05,960 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@782859e OK. 2024-11-18 17:35:05,974 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 17:35:05,975 main DEBUG OutputStream closed 2024-11-18 17:35:05,981 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 17:35:05,981 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 17:35:05,982 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@71a8adcf OK 2024-11-18 17:35:06,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 17:35:06,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 17:35:06,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 17:35:06,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 17:35:06,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 17:35:06,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 17:35:06,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 17:35:06,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 17:35:06,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 17:35:06,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 17:35:06,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 17:35:06,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 17:35:06,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 17:35:06,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 17:35:06,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 17:35:06,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 17:35:06,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 17:35:06,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 17:35:06,307 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 17:35:06,308 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-18 17:35:06,308 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 17:35:06,312 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-18T17:35:07,052 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6 2024-11-18 17:35:07,061 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 17:35:07,062 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T17:35:07,077 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestFSHLog timeout: 13 mins 2024-11-18T17:35:07,141 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970, deleteOnExit=true 2024-11-18T17:35:07,142 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/test.cache.data in system properties and HBase conf 2024-11-18T17:35:07,143 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T17:35:07,149 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir in system properties and HBase conf 2024-11-18T17:35:07,150 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T17:35:07,150 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T17:35:07,151 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-18T17:35:07,493 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T17:35:07,766 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T17:35:07,771 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T17:35:07,772 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T17:35:07,773 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T17:35:07,773 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T17:35:07,774 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T17:35:07,775 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T17:35:07,775 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T17:35:07,776 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T17:35:07,786 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T17:35:07,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/nfs.dump.dir in system properties and HBase conf 2024-11-18T17:35:07,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/java.io.tmpdir in system properties and HBase conf 2024-11-18T17:35:07,800 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T17:35:07,803 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T17:35:07,805 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T17:35:08,955 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-18T17:35:09,624 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T17:35:09,850 INFO [Time-limited test {}] log.Log(170): Logging initialized @6023ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T17:35:10,011 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T17:35:10,150 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T17:35:10,269 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T17:35:10,269 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T17:35:10,271 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T17:35:10,310 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T17:35:10,322 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3837011d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir/,AVAILABLE} 2024-11-18T17:35:10,323 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b6353bb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T17:35:10,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@76669e88{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/java.io.tmpdir/jetty-localhost-32799-hadoop-hdfs-3_4_1-tests_jar-_-any-16755150228061429544/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T17:35:10,777 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75c47eed{HTTP/1.1, (http/1.1)}{localhost:32799} 2024-11-18T17:35:10,778 INFO [Time-limited test {}] server.Server(415): Started @6953ms 2024-11-18T17:35:10,819 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-18T17:35:11,712 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T17:35:11,729 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T17:35:11,769 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T17:35:11,773 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T17:35:11,773 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T17:35:11,780 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ac6a421{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir/,AVAILABLE} 2024-11-18T17:35:11,784 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5815bacf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T17:35:11,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b14cfe0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/java.io.tmpdir/jetty-localhost-32873-hadoop-hdfs-3_4_1-tests_jar-_-any-6647453294192841543/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T17:35:11,993 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1378d4cd{HTTP/1.1, (http/1.1)}{localhost:32873} 2024-11-18T17:35:11,993 INFO [Time-limited test {}] server.Server(415): Started @8168ms 2024-11-18T17:35:12,101 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T17:35:12,439 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T17:35:12,450 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T17:35:12,465 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T17:35:12,465 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T17:35:12,466 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T17:35:12,478 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19945434{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir/,AVAILABLE} 2024-11-18T17:35:12,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fb4c244{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T17:35:12,725 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d3c037d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/java.io.tmpdir/jetty-localhost-43023-hadoop-hdfs-3_4_1-tests_jar-_-any-4598788194556065387/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T17:35:12,733 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30b54a1b{HTTP/1.1, (http/1.1)}{localhost:43023} 2024-11-18T17:35:12,733 INFO [Time-limited test {}] server.Server(415): Started @8908ms 2024-11-18T17:35:12,745 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T17:35:13,042 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T17:35:13,049 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T17:35:13,070 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T17:35:13,070 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T17:35:13,070 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T17:35:13,074 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4eb66c55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir/,AVAILABLE} 2024-11-18T17:35:13,076 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15c9253{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T17:35:13,164 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data1/current/BP-243877644-172.17.0.3-1731951309115/current, will proceed with Du for space computation calculation, 2024-11-18T17:35:13,165 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data3/current/BP-243877644-172.17.0.3-1731951309115/current, will proceed with Du for space computation calculation, 2024-11-18T17:35:13,164 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data4/current/BP-243877644-172.17.0.3-1731951309115/current, will proceed with Du for space computation calculation, 2024-11-18T17:35:13,174 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data2/current/BP-243877644-172.17.0.3-1731951309115/current, will proceed with Du for space computation calculation, 2024-11-18T17:35:13,362 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54d78fce{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/java.io.tmpdir/jetty-localhost-38771-hadoop-hdfs-3_4_1-tests_jar-_-any-17150657145069744447/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T17:35:13,366 WARN [Thread-83 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T17:35:13,367 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T17:35:13,383 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71b13768{HTTP/1.1, (http/1.1)}{localhost:38771} 2024-11-18T17:35:13,383 INFO [Time-limited test {}] server.Server(415): Started @9558ms 2024-11-18T17:35:13,386 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T17:35:13,525 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9a662ef639329263 with lease ID 0x666daae872be05da: Processing first storage report for DS-869638f3-a2be-4949-87b0-7574efdf80c1 from datanode DatanodeRegistration(127.0.0.1:39843, datanodeUuid=492752f5-d29d-4b3f-9dbf-b3398a8fbed7, infoPort=38443, infoSecurePort=0, ipcPort=46445, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115) 2024-11-18T17:35:13,527 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9a662ef639329263 with lease ID 0x666daae872be05da: from storage DS-869638f3-a2be-4949-87b0-7574efdf80c1 node DatanodeRegistration(127.0.0.1:39843, datanodeUuid=492752f5-d29d-4b3f-9dbf-b3398a8fbed7, infoPort=38443, infoSecurePort=0, ipcPort=46445, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-18T17:35:13,528 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d2138f94593b7a3 with lease ID 0x666daae872be05d9: Processing first storage report for DS-768ce038-7c52-4f14-a0f6-aca0c0fde710 from datanode DatanodeRegistration(127.0.0.1:46369, datanodeUuid=aa7ebd67-e5a3-4ae4-9036-63adc181b38c, infoPort=37527, infoSecurePort=0, ipcPort=39265, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115) 2024-11-18T17:35:13,528 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d2138f94593b7a3 with lease ID 0x666daae872be05d9: from storage DS-768ce038-7c52-4f14-a0f6-aca0c0fde710 node DatanodeRegistration(127.0.0.1:46369, datanodeUuid=aa7ebd67-e5a3-4ae4-9036-63adc181b38c, infoPort=37527, infoSecurePort=0, ipcPort=39265, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T17:35:13,528 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9a662ef639329263 with lease ID 0x666daae872be05da: Processing first storage report for DS-c2c757c1-8075-40cc-9439-e1b733aec759 from datanode DatanodeRegistration(127.0.0.1:39843, datanodeUuid=492752f5-d29d-4b3f-9dbf-b3398a8fbed7, infoPort=38443, infoSecurePort=0, ipcPort=46445, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115) 2024-11-18T17:35:13,528 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9a662ef639329263 with lease ID 0x666daae872be05da: from storage DS-c2c757c1-8075-40cc-9439-e1b733aec759 node DatanodeRegistration(127.0.0.1:39843, datanodeUuid=492752f5-d29d-4b3f-9dbf-b3398a8fbed7, infoPort=38443, infoSecurePort=0, ipcPort=46445, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T17:35:13,529 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d2138f94593b7a3 with lease ID 0x666daae872be05d9: Processing first storage report for DS-e9033051-5622-407b-a3ff-af1c3bf325f0 from datanode DatanodeRegistration(127.0.0.1:46369, datanodeUuid=aa7ebd67-e5a3-4ae4-9036-63adc181b38c, infoPort=37527, infoSecurePort=0, ipcPort=39265, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115) 2024-11-18T17:35:13,529 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d2138f94593b7a3 with lease ID 0x666daae872be05d9: from storage DS-e9033051-5622-407b-a3ff-af1c3bf325f0 node DatanodeRegistration(127.0.0.1:46369, datanodeUuid=aa7ebd67-e5a3-4ae4-9036-63adc181b38c, infoPort=37527, infoSecurePort=0, ipcPort=39265, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T17:35:13,834 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data5/current/BP-243877644-172.17.0.3-1731951309115/current, will proceed with Du for space computation calculation, 2024-11-18T17:35:13,865 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data6/current/BP-243877644-172.17.0.3-1731951309115/current, will proceed with Du for space computation calculation, 2024-11-18T17:35:13,965 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T17:35:13,995 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4b8056993c46b038 with lease ID 0x666daae872be05db: Processing first storage report for DS-893706a1-b202-42fe-a55b-e134e83d4c2a from datanode DatanodeRegistration(127.0.0.1:38961, datanodeUuid=ddc3cc5a-81e9-4cf0-8dee-81e29bea8331, infoPort=39085, infoSecurePort=0, ipcPort=39815, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115) 2024-11-18T17:35:13,996 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4b8056993c46b038 with lease ID 0x666daae872be05db: from storage DS-893706a1-b202-42fe-a55b-e134e83d4c2a node DatanodeRegistration(127.0.0.1:38961, datanodeUuid=ddc3cc5a-81e9-4cf0-8dee-81e29bea8331, infoPort=39085, infoSecurePort=0, ipcPort=39815, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T17:35:13,996 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4b8056993c46b038 with lease ID 0x666daae872be05db: Processing first storage report for DS-48ff71a0-5385-4226-8ec4-dcb88ea2d5e2 from datanode DatanodeRegistration(127.0.0.1:38961, datanodeUuid=ddc3cc5a-81e9-4cf0-8dee-81e29bea8331, infoPort=39085, infoSecurePort=0, ipcPort=39815, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115) 2024-11-18T17:35:13,997 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4b8056993c46b038 with lease ID 0x666daae872be05db: from storage DS-48ff71a0-5385-4226-8ec4-dcb88ea2d5e2 node DatanodeRegistration(127.0.0.1:38961, datanodeUuid=ddc3cc5a-81e9-4cf0-8dee-81e29bea8331, infoPort=39085, infoSecurePort=0, ipcPort=39815, storageInfo=lv=-57;cid=testClusterID;nsid=1544701802;c=1731951309115), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T17:35:14,317 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6 2024-11-18T17:35:14,346 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testWALClosureFailureAndCleanup Thread=156, OpenFileDescriptor=390, MaxFileDescriptor=1048576, SystemLoadAverage=1192, ProcessCount=11, AvailableMemoryMB=3801 2024-11-18T17:35:14,410 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:14,428 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:15,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741825_1001 (size=7) 2024-11-18T17:35:15,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741825_1001 (size=7) 2024-11-18T17:35:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741825_1001 (size=7) 2024-11-18T17:35:15,439 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:15,440 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:15,443 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:15,465 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T17:35:15,528 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T17:35:15,539 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:15,549 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs, maxLogs=1760 2024-11-18T17:35:15,671 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951315660 2024-11-18T17:35:15,672 DEBUG [Time-limited test {}] util.CommonFSUtils$DfsBuilderUtility(752): Using builder API via reflection for DFS file creation replicate flag. 2024-11-18T17:35:15,673 DEBUG [Time-limited test {}] util.CommonFSUtils$DfsBuilderUtility(762): Using builder API via reflection for DFS file creation noLocalWrite flag. 2024-11-18T17:35:15,720 WARN [IPC Server handler 4 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-18T17:35:15,721 WARN [IPC Server handler 4 on default port 45055 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-18T17:35:15,721 WARN [IPC Server handler 4 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-18T17:35:15,825 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 2024-11-18T17:35:15,901 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:15,924 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-18T17:35:15,924 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-18T17:35:15,954 INFO [Time-limited test {}] regionserver.HRegion(7106): creating {ENCODED => e240252d4898d12edf6f802727252443, NAME => 'testWALClosureFailureAndCleanup,,1731951315921.e240252d4898d12edf6f802727252443.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testWALClosureFailureAndCleanup', {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6 2024-11-18T17:35:16,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741827_1003 (size=66) 2024-11-18T17:35:16,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741827_1003 (size=66) 2024-11-18T17:35:16,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741827_1003 (size=66) 2024-11-18T17:35:16,097 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T17:35:16,099 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated testWALClosureFailureAndCleanup,,1731951315921.e240252d4898d12edf6f802727252443.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T17:35:16,237 INFO [StoreOpener-e240252d4898d12edf6f802727252443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e240252d4898d12edf6f802727252443 2024-11-18T17:35:16,298 INFO [StoreOpener-e240252d4898d12edf6f802727252443-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e240252d4898d12edf6f802727252443 columnFamilyName b 2024-11-18T17:35:16,307 DEBUG [StoreOpener-e240252d4898d12edf6f802727252443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:16,324 INFO [StoreOpener-e240252d4898d12edf6f802727252443-1 {}] regionserver.HStore(327): Store=e240252d4898d12edf6f802727252443/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:16,348 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testWALClosureFailureAndCleanup/e240252d4898d12edf6f802727252443 2024-11-18T17:35:16,356 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testWALClosureFailureAndCleanup/e240252d4898d12edf6f802727252443 2024-11-18T17:35:16,365 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/e1e38333-24d2-995c-43d7-ad11d6c24901/data/default/testWALClosureFailureAndCleanup/e240252d4898d12edf6f802727252443 2024-11-18T17:35:16,419 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for e240252d4898d12edf6f802727252443 2024-11-18T17:35:16,460 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45055/user/jenkins/test-data/e1e38333-24d2-995c-43d7-ad11d6c24901/data/default/testWALClosureFailureAndCleanup/e240252d4898d12edf6f802727252443/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T17:35:16,473 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened e240252d4898d12edf6f802727252443; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68042022, jitterRate=0.013905137777328491}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T17:35:16,497 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for e240252d4898d12edf6f802727252443: 2024-11-18T17:35:16,561 WARN [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 2 replicas but expecting no less than 3 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:16,561 WARN [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:16,580 WARN [sync.2 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 2 replicas but expecting no less than 3 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:16,581 WARN [sync.2 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:16,592 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951316585 2024-11-18T17:35:16,740 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 123 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:16,745 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 with entries=2, filesize=0 B; new WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 2024-11-18T17:35:16,752 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:16,753 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 is not closed yet, will try archiving it next time 2024-11-18T17:35:16,761 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951316761 2024-11-18T17:35:16,752 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: WAL close injected failure.. at org.apache.hadoop.hbase.regionserver.wal.TestFSHLog$2FailingWriter.close(TestFSHLog.java:351) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:459) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:16,763 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-18T17:35:16,764 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-18T17:35:16,764 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 2024-11-18T17:35:16,770 WARN [IPC Server handler 4 on default port 45055 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 has not been closed. Lease recovery is in progress. RecoveryId = 1005 for block blk_1073741826_1002 2024-11-18T17:35:16,776 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 after 12ms 2024-11-18T17:35:16,885 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 with entries=2, filesize=0 B; new WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316761 2024-11-18T17:35:16,891 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:16,891 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 is not closed yet, will try archiving it next time 2024-11-18T17:35:16,891 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 is not closed yet, will try archiving it next time 2024-11-18T17:35:16,892 WARN [Close-WAL-Writer-1 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: WAL close injected failure.. at org.apache.hadoop.hbase.regionserver.wal.TestFSHLog$2FailingWriter.close(TestFSHLog.java:351) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:459) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:16,896 INFO [Close-WAL-Writer-1 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 2024-11-18T17:35:16,897 WARN [IPC Server handler 3 on default port 45055 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 has not been closed. Lease recovery is in progress. RecoveryId = 1007 for block blk_1073741828_1004 2024-11-18T17:35:16,895 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup 2024-11-18T17:35:16,902 INFO [Close-WAL-Writer-1 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 after 6ms 2024-11-18T17:35:16,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741829_1006 (size=93) 2024-11-18T17:35:16,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741829_1006 (size=93) 2024-11-18T17:35:16,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741829_1006 (size=93) 2024-11-18T17:35:17,457 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:40254 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002] {}] datanode.DataXceiver(331): 127.0.0.1:46369:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40254 dst: /127.0.0.1:46369 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46369 remote=/127.0.0.1:40254]. Total timeout mills is 5000, 3305 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:17,459 WARN [ResponseProcessor for block BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:17,460 WARN [DataStreamer for file /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 block BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002 {}] hdfs.DataStreamer(1731): Error Recovery for BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK]) is bad. 2024-11-18T17:35:17,461 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:56122 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002] {}] datanode.DataXceiver(331): 127.0.0.1:39843:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56122 dst: /127.0.0.1:39843 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:17,496 WARN [ResponseProcessor for block BP-243877644-172.17.0.3-1731951309115:blk_1073741828_1004 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-243877644-172.17.0.3-1731951309115:blk_1073741828_1004 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:17,496 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:40268 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741828_1004] {}] datanode.DataXceiver(331): 127.0.0.1:46369:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40268 dst: /127.0.0.1:46369 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46369 remote=/127.0.0.1:40268]. Total timeout mills is 5000, 4222 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:17,498 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:56150 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741828_1004] {}] datanode.DataXceiver(331): 127.0.0.1:39843:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56150 dst: /127.0.0.1:39843 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:17,500 WARN [DataStreamer for file /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 block BP-243877644-172.17.0.3-1731951309115:blk_1073741828_1004 {}] hdfs.DataStreamer(1731): Error Recovery for BP-243877644-172.17.0.3-1731951309115:blk_1073741828_1004 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK], DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK]) is bad. 2024-11-18T17:35:17,505 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:40514 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741828_1004] {}] datanode.DataXceiver(331): 127.0.0.1:38961:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40514 dst: /127.0.0.1:38961 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:17,509 WARN [IPC Server handler 4 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-18T17:35:17,509 WARN [IPC Server handler 4 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-18T17:35:17,509 WARN [IPC Server handler 4 on default port 45055 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-18T17:35:17,510 WARN [IPC Server handler 4 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-18T17:35:17,511 WARN [DataStreamer for file /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 block BP-243877644-172.17.0.3-1731951309115:blk_1073741828_1004 {}] hdfs.DataStreamer(859): DataStreamer Exception java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK], DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK]], original=[DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK], DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DataStreamer.findNewDatanode(DataStreamer.java:1455) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.addDatanode2ExistingPipeline(DataStreamer.java:1515) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.handleDatanodeReplacement(DataStreamer.java:1758) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1648) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:17,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741828_1007 (size=85) 2024-11-18T17:35:17,531 WARN [DataStreamer for file /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 block BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002 is COMPLETE but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:17,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741828_1007 (size=85) 2024-11-18T17:35:17,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741828_1007 (size=85) 2024-11-18T17:35:20,778 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 after 4014ms 2024-11-18T17:35:20,779 DEBUG [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 is not closed yet, will try archiving it next time 2024-11-18T17:35:20,904 INFO [Close-WAL-Writer-1 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 after 4008ms 2024-11-18T17:35:20,920 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 3 WAL file(s) to /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs 2024-11-18T17:35:20,920 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951316761) 2024-11-18T17:35:20,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testWALClosureFailureAndCleanup Thread=164 (was 156) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@1ac88827[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (96036861) connection to localhost/127.0.0.1:39815 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39265 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45055 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (96036861) connection to localhost/127.0.0.1:39265 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39815 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@664dc259[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=426 (was 390) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1384 (was 1192) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3868 (was 3801) - AvailableMemoryMB LEAK? - 2024-11-18T17:35:20,952 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testSyncRunnerIndexOverflow Thread=164, OpenFileDescriptor=426, MaxFileDescriptor=1048576, SystemLoadAverage=1384, ProcessCount=11, AvailableMemoryMB=3866 2024-11-18T17:35:21,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741830_1008 (size=7) 2024-11-18T17:35:21,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741830_1008 (size=7) 2024-11-18T17:35:21,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741830_1008 (size=7) 2024-11-18T17:35:21,015 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:21,015 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:21,020 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:21,036 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:21,036 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testSyncRunnerIndexOverflow, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs, maxLogs=1760 2024-11-18T17:35:21,041 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951321040 2024-11-18T17:35:21,085 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testSyncRunnerIndexOverflow/wal.1731951321040 2024-11-18T17:35:21,086 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:21,135 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testSyncRunnerIndexOverflow 2024-11-18T17:35:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741831_1009 (size=1293) 2024-11-18T17:35:21,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741831_1009 (size=1293) 2024-11-18T17:35:21,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741831_1009 (size=1293) 2024-11-18T17:35:21,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs 2024-11-18T17:35:21,162 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951321040) 2024-11-18T17:35:21,172 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testSyncRunnerIndexOverflow Thread=164 (was 164), OpenFileDescriptor=426 (was 426), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1384 (was 1384), ProcessCount=11 (was 11), AvailableMemoryMB=3842 (was 3866) 2024-11-18T17:35:21,182 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testDeadlockWithSyncOverwrites Thread=164, OpenFileDescriptor=426, MaxFileDescriptor=1048576, SystemLoadAverage=1384, ProcessCount=11, AvailableMemoryMB=3839 2024-11-18T17:35:21,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741832_1010 (size=7) 2024-11-18T17:35:21,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741832_1010 (size=7) 2024-11-18T17:35:21,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741832_1010 (size=7) 2024-11-18T17:35:21,250 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:21,251 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:21,254 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:21,271 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:21,272 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testDeadlockWithSyncOverwrites, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs, maxLogs=1760 2024-11-18T17:35:21,278 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:21,299 WARN [sync.1 {}] wal.FSHLog$SyncRunner(683): UNEXPECTED java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.wal.WALProvider$Writer.sync(boolean)" because "this.this$0.writer" is null at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:674) ~[classes/:?] 2024-11-18T17:35:21,379 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:21,380 WARN [sync.2 {}] wal.FSHLog$SyncRunner(683): UNEXPECTED java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.wal.WALProvider$Writer.sync(boolean)" because "this.this$0.writer" is null at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:674) ~[classes/:?] 2024-11-18T17:35:21,380 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testDeadlockWithSyncOverwrites 2024-11-18T17:35:21,384 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: CustomFSHLog wal:(num -1) 2024-11-18T17:35:21,394 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testDeadlockWithSyncOverwrites Thread=164 (was 164), OpenFileDescriptor=426 (was 426), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1384 (was 1384), ProcessCount=11 (was 11), AvailableMemoryMB=3806 (was 3839) 2024-11-18T17:35:21,410 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testUnflushedSeqIdTracking Thread=164, OpenFileDescriptor=426, MaxFileDescriptor=1048576, SystemLoadAverage=1384, ProcessCount=11, AvailableMemoryMB=3803 2024-11-18T17:35:21,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741833_1011 (size=7) 2024-11-18T17:35:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741833_1011 (size=7) 2024-11-18T17:35:21,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741833_1011 (size=7) 2024-11-18T17:35:21,856 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:21,857 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:21,859 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:21,868 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:21,868 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testUnflushedSeqIdTracking, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs, maxLogs=1760 2024-11-18T17:35:21,881 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951321881 2024-11-18T17:35:21,992 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testUnflushedSeqIdTracking/wal.1731951321881 2024-11-18T17:35:21,993 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527)] 2024-11-18T17:35:21,994 INFO [Time-limited test {}] regionserver.HRegion(7106): creating {ENCODED => eacee0c2fe3eb57b8f3554dcafed419b, NAME => 'testUnflushedSeqIdTracking,,1731951321994.eacee0c2fe3eb57b8f3554dcafed419b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testUnflushedSeqIdTracking', {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6 2024-11-18T17:35:22,024 WARN [IPC Server handler 4 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-18T17:35:22,024 WARN [IPC Server handler 4 on default port 45055 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-18T17:35:22,024 WARN [IPC Server handler 4 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-18T17:35:22,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741835_1013 (size=61) 2024-11-18T17:35:22,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741835_1013 (size=61) 2024-11-18T17:35:22,510 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated testUnflushedSeqIdTracking,,1731951321994.eacee0c2fe3eb57b8f3554dcafed419b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T17:35:22,529 INFO [StoreOpener-eacee0c2fe3eb57b8f3554dcafed419b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eacee0c2fe3eb57b8f3554dcafed419b 2024-11-18T17:35:22,534 INFO [StoreOpener-eacee0c2fe3eb57b8f3554dcafed419b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eacee0c2fe3eb57b8f3554dcafed419b columnFamilyName b 2024-11-18T17:35:22,534 DEBUG [StoreOpener-eacee0c2fe3eb57b8f3554dcafed419b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:22,543 INFO [StoreOpener-eacee0c2fe3eb57b8f3554dcafed419b-1 {}] regionserver.HStore(327): Store=eacee0c2fe3eb57b8f3554dcafed419b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:22,547 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b 2024-11-18T17:35:22,552 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b 2024-11-18T17:35:22,561 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/abac3b23-b49e-4dd5-19ea-031158c14797/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b 2024-11-18T17:35:22,592 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for eacee0c2fe3eb57b8f3554dcafed419b 2024-11-18T17:35:22,623 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45055/user/jenkins/test-data/abac3b23-b49e-4dd5-19ea-031158c14797/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T17:35:22,625 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened eacee0c2fe3eb57b8f3554dcafed419b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59539671, jitterRate=-0.11278976500034332}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T17:35:22,632 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for eacee0c2fe3eb57b8f3554dcafed419b: 2024-11-18T17:35:23,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741835_1013 (size=61) 2024-11-18T17:35:25,681 INFO [pool-70-thread-2 {}] regionserver.HRegion(2837): Flushing eacee0c2fe3eb57b8f3554dcafed419b 1/1 column families, dataSize=24 B heapSize=352 B 2024-11-18T17:35:28,843 DEBUG [pool-70-thread-2 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b/.tmp/b/671c5a1acbba4f118f327a5c2681dee4 is 28, key is b/b:b/1731951322673/Put/seqid=0 2024-11-18T17:35:28,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741836_1014 (size=4945) 2024-11-18T17:35:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741836_1014 (size=4945) 2024-11-18T17:35:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741836_1014 (size=4945) 2024-11-18T17:35:29,319 INFO [pool-70-thread-2 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=4 (bloomFilter=true), to=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b/.tmp/b/671c5a1acbba4f118f327a5c2681dee4 2024-11-18T17:35:29,489 DEBUG [pool-70-thread-2 {}] regionserver.HRegionFileSystem(442): Committing /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b/.tmp/b/671c5a1acbba4f118f327a5c2681dee4 as /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b/b/671c5a1acbba4f118f327a5c2681dee4 2024-11-18T17:35:29,538 INFO [pool-70-thread-2 {}] regionserver.HStore$StoreFlusherImpl(1989): Added /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/testUnflushedSeqIdTracking/eacee0c2fe3eb57b8f3554dcafed419b/b/671c5a1acbba4f118f327a5c2681dee4, entries=1, sequenceid=4, filesize=4.8 K 2024-11-18T17:35:29,563 INFO [pool-70-thread-2 {}] regionserver.HRegion(3040): Finished flush of dataSize ~48 B/48, heapSize ~432 B/432, currentSize=0 B/0 for eacee0c2fe3eb57b8f3554dcafed419b in 3881ms, sequenceid=4, compaction requested=false 2024-11-18T17:35:29,564 DEBUG [pool-70-thread-2 {}] regionserver.HRegion(2538): Flush status journal for eacee0c2fe3eb57b8f3554dcafed419b: 2024-11-18T17:35:29,564 INFO [pool-70-thread-2 {}] wal.TestFSHLog$4(293): Flush result:FLUSHED_NO_COMPACTION_NEEDED 2024-11-18T17:35:29,564 INFO [pool-70-thread-2 {}] wal.TestFSHLog$4(294): Flush succeeded:true 2024-11-18T17:35:29,568 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing eacee0c2fe3eb57b8f3554dcafed419b, disabling compactions & flushes 2024-11-18T17:35:29,569 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region testUnflushedSeqIdTracking,,1731951321994.eacee0c2fe3eb57b8f3554dcafed419b. 2024-11-18T17:35:29,569 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testUnflushedSeqIdTracking,,1731951321994.eacee0c2fe3eb57b8f3554dcafed419b. 2024-11-18T17:35:29,569 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on testUnflushedSeqIdTracking,,1731951321994.eacee0c2fe3eb57b8f3554dcafed419b. after waiting 0 ms 2024-11-18T17:35:29,569 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region testUnflushedSeqIdTracking,,1731951321994.eacee0c2fe3eb57b8f3554dcafed419b. 2024-11-18T17:35:29,585 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed testUnflushedSeqIdTracking,,1731951321994.eacee0c2fe3eb57b8f3554dcafed419b. 2024-11-18T17:35:29,586 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for eacee0c2fe3eb57b8f3554dcafed419b: 2024-11-18T17:35:29,593 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testUnflushedSeqIdTracking 2024-11-18T17:35:29,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741834_1012 (size=875) 2024-11-18T17:35:29,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741834_1012 (size=875) 2024-11-18T17:35:29,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741834_1012 (size=875) 2024-11-18T17:35:29,660 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs 2024-11-18T17:35:29,661 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951321881) 2024-11-18T17:35:29,677 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testUnflushedSeqIdTracking Thread=170 (was 164) Potentially hanging thread: pool-70-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:51684 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:54418 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-70-thread-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@25a3d4eb java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=416 (was 426), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1521 (was 1384) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4953 (was 3803) - AvailableMemoryMB LEAK? - 2024-11-18T17:35:29,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testWALComparator Thread=170, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=1521, ProcessCount=11, AvailableMemoryMB=4950 2024-11-18T17:35:29,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741837_1015 (size=7) 2024-11-18T17:35:29,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741837_1015 (size=7) 2024-11-18T17:35:29,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741837_1015 (size=7) 2024-11-18T17:35:29,886 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:29,886 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:29,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:29,920 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:29,921 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/testWALComparator, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/oldWALs, maxLogs=1760 2024-11-18T17:35:29,926 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951329925 2024-11-18T17:35:30,052 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/testWALComparator/wal.1731951329925 2024-11-18T17:35:30,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:30,053 DEBUG [Time-limited test {}] wal.AbstractTestFSWAL(214): Log obtained is: FSHLog wal:(num 1731951329925) 2024-11-18T17:35:30,076 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:30,076 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=.meta, logDir=hdfs://localhost:45055/user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/testWALComparator, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/oldWALs, maxLogs=1760 2024-11-18T17:35:30,083 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951330082.meta 2024-11-18T17:35:30,217 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 112 ms, current pipeline: [] 2024-11-18T17:35:30,217 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/testWALComparator/wal.1731951330082.meta 2024-11-18T17:35:30,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:30,228 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/testWALComparator 2024-11-18T17:35:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741838_1016 (size=93) 2024-11-18T17:35:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741838_1016 (size=93) 2024-11-18T17:35:30,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741838_1016 (size=93) 2024-11-18T17:35:30,278 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/oldWALs 2024-11-18T17:35:30,278 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951329925) 2024-11-18T17:35:30,284 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/testWALComparator 2024-11-18T17:35:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741839_1017 (size=93) 2024-11-18T17:35:30,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741839_1017 (size=93) 2024-11-18T17:35:30,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741839_1017 (size=93) 2024-11-18T17:35:30,357 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/663e27e1-c303-e5cc-8041-71ea595b275b/oldWALs 2024-11-18T17:35:30,357 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:.meta(num 1731951330082) 2024-11-18T17:35:30,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testWALComparator Thread=172 (was 170) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=422 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1576 (was 1521) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4906 (was 4950) 2024-11-18T17:35:30,389 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testFindMemStoresEligibleForFlush Thread=172, OpenFileDescriptor=424, MaxFileDescriptor=1048576, SystemLoadAverage=1576, ProcessCount=11, AvailableMemoryMB=4905 2024-11-18T17:35:30,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741840_1018 (size=7) 2024-11-18T17:35:30,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741840_1018 (size=7) 2024-11-18T17:35:30,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741840_1018 (size=7) 2024-11-18T17:35:30,514 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:30,514 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:30,517 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:30,539 DEBUG [Time-limited test {}] wal.AbstractTestFSWAL(382): testFindMemStoresEligibleForFlush 2024-11-18T17:35:30,604 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:30,605 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs, maxLogs=1 2024-11-18T17:35:30,613 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951330612 2024-11-18T17:35:30,704 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330612 2024-11-18T17:35:30,705 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:30,729 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951330729 2024-11-18T17:35:30,821 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330612 with entries=2, filesize=275 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330729 2024-11-18T17:35:30,823 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:30,823 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330612 is not closed yet, will try archiving it next time 2024-11-18T17:35:30,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741841_1019 (size=283) 2024-11-18T17:35:30,848 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951330847 2024-11-18T17:35:30,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741841_1019 (size=283) 2024-11-18T17:35:30,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741841_1019 (size=283) 2024-11-18T17:35:30,968 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 111 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK], DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:30,974 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330729 with entries=2, filesize=275 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330847 2024-11-18T17:35:30,975 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527)] 2024-11-18T17:35:30,975 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330729 is not closed yet, will try archiving it next time 2024-11-18T17:35:30,976 INFO [Time-limited test {}] wal.AbstractFSWAL(718): Too many WALs; count=2, max=1; forcing (partial) flush of 1 region(s): 7e4282b6961433ecf70c9af6607a672a[cf1] 2024-11-18T17:35:30,977 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:30,977 INFO [Time-limited test {}] wal.AbstractFSWAL(718): Too many WALs; count=2, max=1; forcing (partial) flush of 1 region(s): 7e4282b6961433ecf70c9af6607a672a[cf1] 2024-11-18T17:35:30,992 INFO [Time-limited test {}] wal.AbstractFSWAL(718): Too many WALs; count=2, max=1; forcing (partial) flush of 1 region(s): 7e4282b6961433ecf70c9af6607a672a[cf1] 2024-11-18T17:35:30,993 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951330993 2024-11-18T17:35:31,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741842_1020 (size=283) 2024-11-18T17:35:31,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741842_1020 (size=283) 2024-11-18T17:35:31,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741842_1020 (size=283) 2024-11-18T17:35:31,020 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330612 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951330612 2024-11-18T17:35:31,035 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330729 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951330729 2024-11-18T17:35:31,120 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 111 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK], DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK]] 2024-11-18T17:35:31,125 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330847 with entries=2, filesize=275 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330993 2024-11-18T17:35:31,126 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:31,126 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330847 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,126 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:31,127 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951331127 2024-11-18T17:35:31,147 WARN [IPC Server handler 0 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-18T17:35:31,147 WARN [IPC Server handler 0 on default port 45055 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-18T17:35:31,148 WARN [IPC Server handler 0 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-18T17:35:31,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741843_1021 (size=283) 2024-11-18T17:35:31,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741843_1021 (size=283) 2024-11-18T17:35:31,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741843_1021 (size=283) 2024-11-18T17:35:31,160 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330847 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951330847 2024-11-18T17:35:31,209 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330993 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331127 2024-11-18T17:35:31,209 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:31,209 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330993 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,210 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:31,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741844_1022 (size=93) 2024-11-18T17:35:31,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741844_1022 (size=93) 2024-11-18T17:35:31,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741844_1022 (size=93) 2024-11-18T17:35:31,252 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951330993 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951330993 2024-11-18T17:35:31,329 WARN [sync.3 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 2 replicas but expecting no less than 3 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:31,329 WARN [sync.3 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:31,335 WARN [sync.4 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 2 replicas but expecting no less than 3 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:31,336 WARN [sync.4 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:31,336 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951331336 2024-11-18T17:35:31,351 WARN [IPC Server handler 0 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-18T17:35:31,351 WARN [IPC Server handler 0 on default port 45055 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-18T17:35:31,352 WARN [IPC Server handler 0 on default port 45055 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-18T17:35:31,424 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331127 with entries=4, filesize=465 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331336 2024-11-18T17:35:31,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:31,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331127 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,425 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:31,434 WARN [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 2 replicas but expecting no less than 3 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:31,435 WARN [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]] 2024-11-18T17:35:31,442 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951331442 2024-11-18T17:35:31,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741845_1023 (size=473) 2024-11-18T17:35:31,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741845_1023 (size=473) 2024-11-18T17:35:31,536 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331336 with entries=2, filesize=275 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331442 2024-11-18T17:35:31,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:31,541 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331127 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,541 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331336 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,541 INFO [Time-limited test {}] wal.AbstractFSWAL(718): Too many WALs; count=2, max=1; forcing (partial) flush of 2 region(s): 7e4282b6961433ecf70c9af6607a672a[cf1],e97f6de4b6541f87a1436b61a8ea14f9[cf1] 2024-11-18T17:35:31,541 INFO [Time-limited test {}] wal.AbstractFSWAL(718): Too many WALs; count=2, max=1; forcing (partial) flush of 2 region(s): 7e4282b6961433ecf70c9af6607a672a[cf1],e97f6de4b6541f87a1436b61a8ea14f9[cf1] 2024-11-18T17:35:31,542 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951331542 2024-11-18T17:35:31,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741846_1024 (size=283) 2024-11-18T17:35:31,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741846_1024 (size=283) 2024-11-18T17:35:31,570 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331127 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,571 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331336 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951331336 2024-11-18T17:35:31,651 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 101 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK], DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK]] 2024-11-18T17:35:31,656 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331442 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331542 2024-11-18T17:35:31,657 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:37527:37527)] 2024-11-18T17:35:31,657 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331127 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,657 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331442 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,657 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:31,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741847_1025 (size=93) 2024-11-18T17:35:31,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741847_1025 (size=93) 2024-11-18T17:35:31,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741847_1025 (size=93) 2024-11-18T17:35:31,693 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331127 is not closed yet, will try archiving it next time 2024-11-18T17:35:31,700 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331442 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951331442 2024-11-18T17:35:31,862 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331127 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951331127 2024-11-18T17:35:31,964 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951331963 2024-11-18T17:35:32,063 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331542 with entries=2, filesize=275 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331963 2024-11-18T17:35:32,063 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:37527:37527)] 2024-11-18T17:35:32,064 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331542 is not closed yet, will try archiving it next time 2024-11-18T17:35:32,064 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:32,065 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951332064 2024-11-18T17:35:32,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741848_1026 (size=283) 2024-11-18T17:35:32,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741848_1026 (size=283) 2024-11-18T17:35:32,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741848_1026 (size=283) 2024-11-18T17:35:32,100 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331542 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951331542 2024-11-18T17:35:32,153 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331963 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951332064 2024-11-18T17:35:32,154 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527)] 2024-11-18T17:35:32,154 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331963 is not closed yet, will try archiving it next time 2024-11-18T17:35:32,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741849_1027 (size=93) 2024-11-18T17:35:32,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741849_1027 (size=93) 2024-11-18T17:35:32,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741849_1027 (size=93) 2024-11-18T17:35:32,180 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951331963 to hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs/wal.1731951331963 2024-11-18T17:35:32,213 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951332212 2024-11-18T17:35:32,324 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951332064 with entries=6, filesize=709 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951332212 2024-11-18T17:35:32,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:32,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951332064 is not closed yet, will try archiving it next time 2024-11-18T17:35:32,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741850_1028 (size=717) 2024-11-18T17:35:32,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741850_1028 (size=717) 2024-11-18T17:35:32,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741850_1028 (size=717) 2024-11-18T17:35:32,343 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951332342 2024-11-18T17:35:32,425 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951332212 with entries=2, filesize=293 B; new WAL /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951332342 2024-11-18T17:35:32,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:32,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45055/user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush/wal.1731951332212 is not closed yet, will try archiving it next time 2024-11-18T17:35:32,425 INFO [Time-limited test {}] wal.AbstractFSWAL(718): Too many WALs; count=2, max=1; forcing (partial) flush of 1 region(s): 63bc982cbe4f0a17baa23ac6d6a466fe[cf1,cf3,cf2] 2024-11-18T17:35:32,426 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T17:35:32,426 INFO [Time-limited test {}] wal.AbstractFSWAL(718): Too many WALs; count=2, max=1; forcing (partial) flush of 1 region(s): 63bc982cbe4f0a17baa23ac6d6a466fe[cf3,cf2] 2024-11-18T17:35:32,433 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/testFindMemStoresEligibleForFlush 2024-11-18T17:35:32,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741851_1029 (size=301) 2024-11-18T17:35:32,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741851_1029 (size=301) 2024-11-18T17:35:32,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741851_1029 (size=301) 2024-11-18T17:35:32,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741852_1030 (size=93) 2024-11-18T17:35:32,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741852_1030 (size=93) 2024-11-18T17:35:32,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741852_1030 (size=93) 2024-11-18T17:35:32,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741845_1023 (size=473) 2024-11-18T17:35:32,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741846_1024 (size=283) 2024-11-18T17:35:32,517 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 3 WAL file(s) to /user/jenkins/test-data/ed4b9935-18cb-d50f-535b-5cd37b896a46/oldWALs 2024-11-18T17:35:32,517 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951332342) 2024-11-18T17:35:32,542 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testFindMemStoresEligibleForFlush Thread=173 (was 172) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@2ea7514c[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=422 (was 424), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1576 (was 1576), ProcessCount=11 (was 11), AvailableMemoryMB=4788 (was 4905) 2024-11-18T17:35:32,562 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testMaxFlushedSequenceIdGoBackwards Thread=173, OpenFileDescriptor=422, MaxFileDescriptor=1048576, SystemLoadAverage=1576, ProcessCount=11, AvailableMemoryMB=4785 2024-11-18T17:35:32,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741853_1031 (size=7) 2024-11-18T17:35:32,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741853_1031 (size=7) 2024-11-18T17:35:32,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741853_1031 (size=7) 2024-11-18T17:35:32,698 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:32,698 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:32,702 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:32,729 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:32,729 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testMaxFlushedSequenceIdGoBackwards, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs, maxLogs=1760 2024-11-18T17:35:32,733 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951332732 2024-11-18T17:35:32,834 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testMaxFlushedSequenceIdGoBackwards/wal.1731951332732 2024-11-18T17:35:32,834 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:32,835 INFO [Time-limited test {}] regionserver.HRegion(7106): creating {ENCODED => c6f7eb0f5ff075e101014a38c1ab7a91, NAME => 'table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6 2024-11-18T17:35:32,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741855_1033 (size=40) 2024-11-18T17:35:32,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741855_1033 (size=40) 2024-11-18T17:35:32,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741855_1033 (size=40) 2024-11-18T17:35:32,957 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T17:35:32,978 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:32,993 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6f7eb0f5ff075e101014a38c1ab7a91 columnFamilyName a 2024-11-18T17:35:32,994 DEBUG [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:33,002 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] regionserver.HStore(327): Store=c6f7eb0f5ff075e101014a38c1ab7a91/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:33,002 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:33,014 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6f7eb0f5ff075e101014a38c1ab7a91 columnFamilyName b 2024-11-18T17:35:33,014 DEBUG [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:33,025 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] regionserver.HStore(327): Store=c6f7eb0f5ff075e101014a38c1ab7a91/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:33,033 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:33,036 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:33,039 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/fcb3f596-c279-67d7-d240-12952bba4ce7/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:33,053 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-18T17:35:33,068 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:33,100 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45055/user/jenkins/test-data/fcb3f596-c279-67d7-d240-12952bba4ce7/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T17:35:33,102 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened c6f7eb0f5ff075e101014a38c1ab7a91; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59563771, jitterRate=-0.1124306470155716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-18T17:35:33,106 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for c6f7eb0f5ff075e101014a38c1ab7a91: 2024-11-18T17:35:33,106 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing c6f7eb0f5ff075e101014a38c1ab7a91, disabling compactions & flushes 2024-11-18T17:35:33,106 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. 2024-11-18T17:35:33,106 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. 2024-11-18T17:35:33,106 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. after waiting 0 ms 2024-11-18T17:35:33,107 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. 2024-11-18T17:35:33,116 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. 2024-11-18T17:35:33,117 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for c6f7eb0f5ff075e101014a38c1ab7a91: 2024-11-18T17:35:34,198 DEBUG [Time-limited test {}] regionserver.HRegion(7285): Opening region: {ENCODED => c6f7eb0f5ff075e101014a38c1ab7a91, NAME => 'table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91.', STARTKEY => '', ENDKEY => ''} 2024-11-18T17:35:34,250 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,251 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T17:35:34,254 DEBUG [Time-limited test {}] regionserver.HRegion(7327): checking encryption for c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,255 DEBUG [Time-limited test {}] regionserver.HRegion(7330): checking classloading for c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,293 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,301 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6f7eb0f5ff075e101014a38c1ab7a91 columnFamilyName a 2024-11-18T17:35:34,301 DEBUG [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:34,316 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] regionserver.HStore(327): Store=c6f7eb0f5ff075e101014a38c1ab7a91/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:34,317 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,328 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6f7eb0f5ff075e101014a38c1ab7a91 columnFamilyName b 2024-11-18T17:35:34,329 DEBUG [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:34,347 INFO [StoreOpener-c6f7eb0f5ff075e101014a38c1ab7a91-1 {}] regionserver.HStore(327): Store=c6f7eb0f5ff075e101014a38c1ab7a91/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:34,356 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,365 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,409 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/fcb3f596-c279-67d7-d240-12952bba4ce7/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,444 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for c6f7eb0f5ff075e101014a38c1ab7a91 2024-11-18T17:35:34,448 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened c6f7eb0f5ff075e101014a38c1ab7a91; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71114548, jitterRate=0.059689342975616455}}}, org.apache.hadoop.hbase.regionserver.wal.AbstractTestFSWAL$FlushSpecificStoresPolicy@5fbcff27 2024-11-18T17:35:34,452 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for c6f7eb0f5ff075e101014a38c1ab7a91: 2024-11-18T17:35:37,554 INFO [pool-84-thread-1 {}] regionserver.HRegion(2837): Flushing c6f7eb0f5ff075e101014a38c1ab7a91 2/2 column families, dataSize=96 B heapSize=896 B 2024-11-18T17:35:38,833 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.WAL.CP_org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor 2024-11-18T17:35:38,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.WAL.CP_org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor Metrics about HBase WALObservers 2024-11-18T17:35:38,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testWALClosureFailureAndCleanup 2024-11-18T17:35:38,836 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testWALClosureFailureAndCleanup Metrics about Tables on a single HBase RegionServer 2024-11-18T17:35:38,850 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table 2024-11-18T17:35:38,850 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table Metrics about Tables on a single HBase RegionServer 2024-11-18T17:35:39,801 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T17:35:40,620 DEBUG [pool-84-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/a/b83242adb0824b60b899c60c29a591f4 is 28, key is a/a:a/1731951334497/Put/seqid=0 2024-11-18T17:35:40,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741856_1034 (size=4945) 2024-11-18T17:35:40,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741856_1034 (size=4945) 2024-11-18T17:35:40,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741856_1034 (size=4945) 2024-11-18T17:35:40,724 INFO [pool-84-thread-1 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=6 (bloomFilter=true), to=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/a/b83242adb0824b60b899c60c29a591f4 2024-11-18T17:35:40,794 DEBUG [pool-84-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/b/d0a56b91aa724c1eb4177e93a0d5e07e is 28, key is a/b:b/1731951334497/Put/seqid=0 2024-11-18T17:35:40,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741857_1035 (size=4945) 2024-11-18T17:35:40,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741857_1035 (size=4945) 2024-11-18T17:35:40,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741857_1035 (size=4945) 2024-11-18T17:35:40,892 INFO [pool-84-thread-1 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=6 (bloomFilter=true), to=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/b/d0a56b91aa724c1eb4177e93a0d5e07e 2024-11-18T17:35:40,948 DEBUG [pool-84-thread-1 {}] regionserver.HRegionFileSystem(442): Committing /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/a/b83242adb0824b60b899c60c29a591f4 as /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/a/b83242adb0824b60b899c60c29a591f4 2024-11-18T17:35:40,993 INFO [pool-84-thread-1 {}] regionserver.HStore$StoreFlusherImpl(1989): Added /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/a/b83242adb0824b60b899c60c29a591f4, entries=1, sequenceid=6, filesize=4.8 K 2024-11-18T17:35:41,000 DEBUG [pool-84-thread-1 {}] regionserver.HRegionFileSystem(442): Committing /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/b/d0a56b91aa724c1eb4177e93a0d5e07e as /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/b/d0a56b91aa724c1eb4177e93a0d5e07e 2024-11-18T17:35:41,067 INFO [pool-84-thread-1 {}] regionserver.HStore$StoreFlusherImpl(1989): Added /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/b/d0a56b91aa724c1eb4177e93a0d5e07e, entries=1, sequenceid=6, filesize=4.8 K 2024-11-18T17:35:41,080 INFO [pool-84-thread-1 {}] regionserver.HRegion(3040): Finished flush of dataSize ~96 B/96, heapSize ~864 B/864, currentSize=0 B/0 for c6f7eb0f5ff075e101014a38c1ab7a91 in 3527ms, sequenceid=6, compaction requested=false 2024-11-18T17:35:41,081 DEBUG [pool-84-thread-1 {}] regionserver.HRegion(2538): Flush status journal for c6f7eb0f5ff075e101014a38c1ab7a91: 2024-11-18T17:35:41,081 INFO [pool-84-thread-1 {}] wal.AbstractTestFSWAL(657): Flush result:FLUSHED_NO_COMPACTION_NEEDED 2024-11-18T17:35:41,081 INFO [pool-84-thread-1 {}] wal.AbstractTestFSWAL(658): Flush succeeded:true 2024-11-18T17:35:41,093 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing c6f7eb0f5ff075e101014a38c1ab7a91 1/2 column families, dataSize=48 B heapSize=704 B; a={dataSize=24 B, heapSize=352 B, offHeapSize=0 B} 2024-11-18T17:35:41,123 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/a/64dcbc1c598f46d69908000df39ce24f is 28, key is a/a:a/1731951334497/Put/seqid=0 2024-11-18T17:35:41,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741858_1036 (size=4945) 2024-11-18T17:35:41,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741858_1036 (size=4945) 2024-11-18T17:35:41,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741858_1036 (size=4945) 2024-11-18T17:35:41,253 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24 B at sequenceid=10 (bloomFilter=true), to=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/a/64dcbc1c598f46d69908000df39ce24f 2024-11-18T17:35:41,304 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/a/64dcbc1c598f46d69908000df39ce24f as /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/a/64dcbc1c598f46d69908000df39ce24f 2024-11-18T17:35:41,353 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/a/64dcbc1c598f46d69908000df39ce24f, entries=1, sequenceid=10, filesize=4.8 K 2024-11-18T17:35:41,365 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~24 B/24, heapSize ~336 B/336, currentSize=24 B/24 for c6f7eb0f5ff075e101014a38c1ab7a91 in 272ms, sequenceid=10, compaction requested=false 2024-11-18T17:35:41,366 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for c6f7eb0f5ff075e101014a38c1ab7a91: 2024-11-18T17:35:41,366 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing c6f7eb0f5ff075e101014a38c1ab7a91, disabling compactions & flushes 2024-11-18T17:35:41,366 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. 2024-11-18T17:35:41,366 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. 2024-11-18T17:35:41,367 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. after waiting 0 ms 2024-11-18T17:35:41,367 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. 2024-11-18T17:35:41,367 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing c6f7eb0f5ff075e101014a38c1ab7a91 2/2 column families, dataSize=24 B heapSize=608 B 2024-11-18T17:35:41,395 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/b/19ff2d0bb1ff45d6a3a1da62c598fcf0 is 28, key is a/b:b/1731951334497/Put/seqid=0 2024-11-18T17:35:41,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741859_1037 (size=4945) 2024-11-18T17:35:41,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741859_1037 (size=4945) 2024-11-18T17:35:41,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741859_1037 (size=4945) 2024-11-18T17:35:41,509 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24 B at sequenceid=13 (bloomFilter=true), to=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/b/19ff2d0bb1ff45d6a3a1da62c598fcf0 2024-11-18T17:35:41,561 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/.tmp/b/19ff2d0bb1ff45d6a3a1da62c598fcf0 as /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/b/19ff2d0bb1ff45d6a3a1da62c598fcf0 2024-11-18T17:35:41,602 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/b/19ff2d0bb1ff45d6a3a1da62c598fcf0, entries=1, sequenceid=13, filesize=4.8 K 2024-11-18T17:35:41,624 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~24 B/24, heapSize ~336 B/336, currentSize=0 B/0 for c6f7eb0f5ff075e101014a38c1ab7a91 in 257ms, sequenceid=13, compaction requested=false 2024-11-18T17:35:41,690 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45055/user/jenkins/test-data/fcb3f596-c279-67d7-d240-12952bba4ce7/data/default/table/c6f7eb0f5ff075e101014a38c1ab7a91/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-18T17:35:41,692 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed table,,1731951332834.c6f7eb0f5ff075e101014a38c1ab7a91. 2024-11-18T17:35:41,692 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for c6f7eb0f5ff075e101014a38c1ab7a91: 2024-11-18T17:35:41,693 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testMaxFlushedSequenceIdGoBackwards 2024-11-18T17:35:41,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741854_1032 (size=2357) 2024-11-18T17:35:41,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741854_1032 (size=2357) 2024-11-18T17:35:41,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741854_1032 (size=2357) 2024-11-18T17:35:41,753 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs 2024-11-18T17:35:41,753 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951332732) 2024-11-18T17:35:41,765 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testMaxFlushedSequenceIdGoBackwards Thread=174 (was 173) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:59140 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:33404 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:33332 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=424 (was 422) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1770 (was 1576) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4414 (was 4785) 2024-11-18T17:35:41,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testFlushSequenceIdIsGreaterThanAllEditsInHFile Thread=174, OpenFileDescriptor=424, MaxFileDescriptor=1048576, SystemLoadAverage=1770, ProcessCount=11, AvailableMemoryMB=4412 2024-11-18T17:35:41,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741860_1038 (size=7) 2024-11-18T17:35:41,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741860_1038 (size=7) 2024-11-18T17:35:41,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741860_1038 (size=7) 2024-11-18T17:35:41,897 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:41,898 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:41,900 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:41,921 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.HBaseTestingUtility.createWal(HBaseTestingUtility.java:2587) ~[test-classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtility.createRegionAndWAL(HBaseTestingUtility.java:2632) ~[test-classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtility.createRegionAndWAL(HBaseTestingUtility.java:2596) ~[test-classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestFSWAL.testFlushSequenceIdIsGreaterThanAllEditsInHFile(AbstractTestFSWAL.java:424) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:41,922 INFO [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T17:35:41,927 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T17:35:41,944 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T17:35:41,944 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T17:35:41,972 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T17:35:41,984 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:41,984 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=hregion-72602810, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/WALs/hregion-72602810, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/oldWALs, maxLogs=1760 2024-11-18T17:35:42,021 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/WALs/hregion-72602810/hregion-72602810.1731951341990, exclude list is [], retry=0 2024-11-18T17:35:42,058 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 889 (auto-detected) 2024-11-18T17:35:42,062 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-18T17:35:42,101 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK] 2024-11-18T17:35:42,106 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T17:35:42,132 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK] 2024-11-18T17:35:42,134 DEBUG [AsyncFSWAL-1-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK] 2024-11-18T17:35:42,178 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/WALs/hregion-72602810/hregion-72602810.1731951341990 2024-11-18T17:35:42,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:42,181 INFO [Time-limited test {}] regionserver.HRegion(7106): creating {ENCODED => c1e136f1c8ca00c03738d7b891cf82ab, NAME => 'testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testFlushSequenceIdIsGreaterThanAllEditsInHFile', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 2024-11-18T17:35:42,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741862_1040 (size=82) 2024-11-18T17:35:42,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741862_1040 (size=82) 2024-11-18T17:35:42,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741862_1040 (size=82) 2024-11-18T17:35:42,285 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T17:35:42,289 INFO [StoreOpener-c1e136f1c8ca00c03738d7b891cf82ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,297 INFO [StoreOpener-c1e136f1c8ca00c03738d7b891cf82ab-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e136f1c8ca00c03738d7b891cf82ab columnFamilyName f 2024-11-18T17:35:42,297 DEBUG [StoreOpener-c1e136f1c8ca00c03738d7b891cf82ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:42,298 INFO [StoreOpener-c1e136f1c8ca00c03738d7b891cf82ab-1 {}] regionserver.HStore(327): Store=c1e136f1c8ca00c03738d7b891cf82ab/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:42,300 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,302 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,303 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,308 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,313 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45055/user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T17:35:42,314 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened c1e136f1c8ca00c03738d7b891cf82ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70672547, jitterRate=0.0531030148267746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T17:35:42,316 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for c1e136f1c8ca00c03738d7b891cf82ab: 2024-11-18T17:35:42,316 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing c1e136f1c8ca00c03738d7b891cf82ab, disabling compactions & flushes 2024-11-18T17:35:42,316 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. 2024-11-18T17:35:42,316 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. 2024-11-18T17:35:42,316 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. after waiting 0 ms 2024-11-18T17:35:42,316 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. 2024-11-18T17:35:42,317 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. 2024-11-18T17:35:42,317 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for c1e136f1c8ca00c03738d7b891cf82ab: 2024-11-18T17:35:42,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741861_1039 (size=93) 2024-11-18T17:35:42,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741861_1039 (size=93) 2024-11-18T17:35:42,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741861_1039 (size=93) 2024-11-18T17:35:42,338 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/oldWALs 2024-11-18T17:35:42,338 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL hregion-72602810:(num 1731951341990) 2024-11-18T17:35:42,348 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:42,349 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/testFlushSequenceIdIsGreaterThanAllEditsInHFile, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/testFlushSequenceIdIsGreaterThanAllEditsInHFile, maxLogs=1760 2024-11-18T17:35:42,353 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951342352 2024-11-18T17:35:42,386 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/testFlushSequenceIdIsGreaterThanAllEditsInHFile/wal.1731951342352 2024-11-18T17:35:42,386 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:42,387 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:42,389 DEBUG [Time-limited test {}] regionserver.HRegion(7285): Opening region: {ENCODED => c1e136f1c8ca00c03738d7b891cf82ab, NAME => 'testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab.', STARTKEY => '', ENDKEY => ''} 2024-11-18T17:35:42,389 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T17:35:42,389 DEBUG [Time-limited test {}] regionserver.HRegion(7327): checking encryption for c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,389 DEBUG [Time-limited test {}] regionserver.HRegion(7330): checking classloading for c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,394 INFO [StoreOpener-c1e136f1c8ca00c03738d7b891cf82ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,396 INFO [StoreOpener-c1e136f1c8ca00c03738d7b891cf82ab-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e136f1c8ca00c03738d7b891cf82ab columnFamilyName f 2024-11-18T17:35:42,396 DEBUG [StoreOpener-c1e136f1c8ca00c03738d7b891cf82ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:42,398 INFO [StoreOpener-c1e136f1c8ca00c03738d7b891cf82ab-1 {}] regionserver.HStore(327): Store=c1e136f1c8ca00c03738d7b891cf82ab/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:42,400 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,401 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,403 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,410 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for c1e136f1c8ca00c03738d7b891cf82ab 2024-11-18T17:35:42,411 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened c1e136f1c8ca00c03738d7b891cf82ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65198087, jitterRate=-0.028472796082496643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T17:35:42,413 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for c1e136f1c8ca00c03738d7b891cf82ab: 2024-11-18T17:35:42,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir so I do NOT create it in target/test-data/d9cb7b84-d667-c01b-1ffe-c8c3ddad4d63 2024-11-18T17:35:42,433 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir Erasing configuration value by system value. 2024-11-18T17:35:42,433 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.tmp.dir so I do NOT create it in target/test-data/d9cb7b84-d667-c01b-1ffe-c8c3ddad4d63 2024-11-18T17:35:42,433 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.tmp.dir Erasing configuration value by system value. 2024-11-18T17:35:42,433 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d9cb7b84-d667-c01b-1ffe-c8c3ddad4d63 2024-11-18T17:35:42,471 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing c1e136f1c8ca00c03738d7b891cf82ab 1/1 column families, dataSize=1.14 KB heapSize=2.13 KB 2024-11-18T17:35:42,572 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:42,673 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:42,774 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:42,874 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:42,975 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:43,076 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:43,176 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:43,277 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:43,378 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:43,478 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:43,579 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:43,611 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab/.tmp/f/f0c172dfc13e4f7e93936f83aadf2e10 is 121, key is testFlushSequenceIdIsGreaterThanAllEditsInHFile/f:x0/1731951342433/Put/seqid=0 2024-11-18T17:35:43,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741864_1042 (size=6333) 2024-11-18T17:35:43,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741864_1042 (size=6333) 2024-11-18T17:35:43,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741864_1042 (size=6333) 2024-11-18T17:35:43,673 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab/.tmp/f/f0c172dfc13e4f7e93936f83aadf2e10 2024-11-18T17:35:43,688 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab/.tmp/f/f0c172dfc13e4f7e93936f83aadf2e10 as hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab/f/f0c172dfc13e4f7e93936f83aadf2e10 2024-11-18T17:35:43,704 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/testFlushSequenceIdIsGreaterThanAllEditsInHFile/c1e136f1c8ca00c03738d7b891cf82ab/f/f0c172dfc13e4f7e93936f83aadf2e10, entries=10, sequenceid=23, filesize=6.2 K 2024-11-18T17:35:43,806 DEBUG [Time-limited test.append-pool-0 {}] wal.AbstractTestFSWAL$1(441): Sleeping before appending 100ms 2024-11-18T17:35:43,808 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~1.14 KB/1170, heapSize ~2.11 KB/2160, currentSize=0 B/0 for c1e136f1c8ca00c03738d7b891cf82ab in 1337ms, sequenceid=23, compaction requested=false 2024-11-18T17:35:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for c1e136f1c8ca00c03738d7b891cf82ab: 2024-11-18T17:35:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing c1e136f1c8ca00c03738d7b891cf82ab, disabling compactions & flushes 2024-11-18T17:35:43,808 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. 2024-11-18T17:35:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. 2024-11-18T17:35:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. after waiting 0 ms 2024-11-18T17:35:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. 2024-11-18T17:35:43,812 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed testFlushSequenceIdIsGreaterThanAllEditsInHFile,,1731951341910.c1e136f1c8ca00c03738d7b891cf82ab. 2024-11-18T17:35:43,812 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for c1e136f1c8ca00c03738d7b891cf82ab: 2024-11-18T17:35:43,813 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/testFlushSequenceIdIsGreaterThanAllEditsInHFile 2024-11-18T17:35:43,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741863_1041 (size=16547) 2024-11-18T17:35:43,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741863_1041 (size=16547) 2024-11-18T17:35:43,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741863_1041 (size=16547) 2024-11-18T17:35:43,838 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/6c543c1b-46d9-5a67-5953-aa262686b9b4/testFlushSequenceIdIsGreaterThanAllEditsInHFile 2024-11-18T17:35:43,838 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: wal:(num 1731951342352) 2024-11-18T17:35:43,851 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testFlushSequenceIdIsGreaterThanAllEditsInHFile Thread=175 (was 174) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:59140 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-1-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-917862887_22 at /127.0.0.1:33404 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=426 (was 424) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1770 (was 1770), ProcessCount=11 (was 11), AvailableMemoryMB=4229 (was 4412) 2024-11-18T17:35:43,862 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testFailedToCreateWALIfParentRenamed Thread=175, OpenFileDescriptor=426, MaxFileDescriptor=1048576, SystemLoadAverage=1770, ProcessCount=11, AvailableMemoryMB=4227 2024-11-18T17:35:43,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741865_1043 (size=7) 2024-11-18T17:35:43,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741865_1043 (size=7) 2024-11-18T17:35:43,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741865_1043 (size=7) 2024-11-18T17:35:44,313 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T17:35:44,326 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:44,326 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:44,329 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:44,340 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:44,340 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/oldWALs, maxLogs=1760 2024-11-18T17:35:44,341 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951344341 2024-11-18T17:35:44,443 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344341 2024-11-18T17:35:44,443 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:44,444 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951344443 2024-11-18T17:35:44,541 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951344444 2024-11-18T17:35:44,548 WARN [Time-limited test {}] wal.AbstractProtobufLogWriter(199): Init output failed, path=hdfs://localhost:45055/user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344444 java.io.FileNotFoundException: Parent directory doesn't exist: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed at org.apache.hadoop.hdfs.server.namenode.FSDirectory.verifyParentDir(FSDirectory.java:2037) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.resolvePathForStartFile(FSDirWriteFileOp.java:338) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2773) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2713) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:830) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:504) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:300) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1300) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$10.doCall(DistributedFileSystem.java:674) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$10.doCall(DistributedFileSystem.java:671) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.createNonRecursive(DistributedFileSystem.java:692) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.access$500(DistributedFileSystem.java:148) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$HdfsDataOutputStreamBuilder.build(DistributedFileSystem.java:3873) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hbase.util.CommonFSUtils.createForWal(CommonFSUtils.java:832) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:103) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:81) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:307) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:70) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestFSWAL.testFailedToCreateWALIfParentRenamed(AbstractTestFSWAL.java:405) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.ExpectException.evaluate(ExpectException.java:19) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Parent directory doesn't exist: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed at org.apache.hadoop.hdfs.server.namenode.FSDirectory.verifyParentDir(FSDirectory.java:2037) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.resolvePathForStartFile(FSDirWriteFileOp.java:338) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2773) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2713) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:830) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:504) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.create(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$create$2(ClientNamenodeProtocolTranslatorPB.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:294) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 42 more 2024-11-18T17:35:44,551 DEBUG [Time-limited test {}] wal.FSHLogProvider(92): Error instantiating log writer. java.io.FileNotFoundException: Parent directory doesn't exist: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed at org.apache.hadoop.hdfs.server.namenode.FSDirectory.verifyParentDir(FSDirectory.java:2037) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.resolvePathForStartFile(FSDirWriteFileOp.java:338) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2773) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2713) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:830) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:504) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:300) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1300) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$10.doCall(DistributedFileSystem.java:674) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$10.doCall(DistributedFileSystem.java:671) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.createNonRecursive(DistributedFileSystem.java:692) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.access$500(DistributedFileSystem.java:148) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$HdfsDataOutputStreamBuilder.build(DistributedFileSystem.java:3873) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hbase.util.CommonFSUtils.createForWal(CommonFSUtils.java:832) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:103) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:81) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:307) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:70) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestFSWAL.testFailedToCreateWALIfParentRenamed(AbstractTestFSWAL.java:405) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.ExpectException.evaluate(ExpectException.java:19) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Parent directory doesn't exist: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed at org.apache.hadoop.hdfs.server.namenode.FSDirectory.verifyParentDir(FSDirectory.java:2037) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.resolvePathForStartFile(FSDirWriteFileOp.java:338) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2773) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2713) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:830) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:504) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.create(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$create$2(ClientNamenodeProtocolTranslatorPB.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:381) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.create(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:294) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 42 more 2024-11-18T17:35:44,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testFailedToCreateWALIfParentRenamed Thread=197 (was 175) Potentially hanging thread: PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=2:[127.0.0.1:46369, 127.0.0.1:39843] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1420) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:33404 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:650) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:59218 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:650) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344341 block BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344443 block BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: ResponseProcessor for block BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) app//org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) Potentially hanging thread: ResponseProcessor for block BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) app//org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) Potentially hanging thread: PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:650) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:650) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:33496 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46369] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1420) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:33508 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:59216 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39843] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1420) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.append-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=2:[127.0.0.1:38961, 127.0.0.1:46369] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) java.base@17.0.11/java.io.FilterInputStream.read(FilterInputStream.java:82) app//org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) app//org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1420) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:650) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1869097489_22 at /127.0.0.1:33418 [Receiving block BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 426) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1770 (was 1770), ProcessCount=11 (was 11), AvailableMemoryMB=4118 (was 4227) 2024-11-18T17:35:44,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testWALCoprocessorLoaded Thread=197, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=1770, ProcessCount=11, AvailableMemoryMB=4112 2024-11-18T17:35:44,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741868_1046 (size=7) 2024-11-18T17:35:44,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741868_1046 (size=7) 2024-11-18T17:35:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741868_1046 (size=7) 2024-11-18T17:35:44,715 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:44,715 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:44,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:44,739 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:44,739 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/c48b12b8-f3c5-9cfe-9b95-9430769874f2/testWALCoprocessorLoaded, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/c48b12b8-f3c5-9cfe-9b95-9430769874f2/oldWALs, maxLogs=1760 2024-11-18T17:35:44,743 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951344743 2024-11-18T17:35:44,809 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c48b12b8-f3c5-9cfe-9b95-9430769874f2/testWALCoprocessorLoaded/wal.1731951344743 2024-11-18T17:35:44,809 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527)] 2024-11-18T17:35:44,820 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/c48b12b8-f3c5-9cfe-9b95-9430769874f2/testWALCoprocessorLoaded 2024-11-18T17:35:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741869_1047 (size=93) 2024-11-18T17:35:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741869_1047 (size=93) 2024-11-18T17:35:44,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741869_1047 (size=93) 2024-11-18T17:35:44,854 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/c48b12b8-f3c5-9cfe-9b95-9430769874f2/oldWALs 2024-11-18T17:35:44,854 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951344743) 2024-11-18T17:35:44,869 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testWALCoprocessorLoaded Thread=197 (was 197), OpenFileDescriptor=464 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1770 (was 1770), ProcessCount=11 (was 11), AvailableMemoryMB=4081 (was 4112) 2024-11-18T17:35:44,884 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testSyncNoAppend Thread=197, OpenFileDescriptor=464, MaxFileDescriptor=1048576, SystemLoadAverage=1770, ProcessCount=11, AvailableMemoryMB=4078 2024-11-18T17:35:45,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741870_1048 (size=7) 2024-11-18T17:35:45,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741870_1048 (size=7) 2024-11-18T17:35:45,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741870_1048 (size=7) 2024-11-18T17:35:45,016 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:45,017 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:45,019 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:45,026 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:45,028 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/d6270473-361d-febe-1021-c7fb9d4c3b3e/testSyncNoAppend, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/d6270473-361d-febe-1021-c7fb9d4c3b3e/testSyncNoAppend, maxLogs=1760 2024-11-18T17:35:45,030 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951345030 2024-11-18T17:35:45,162 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 117 ms, current pipeline: [] 2024-11-18T17:35:45,163 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/d6270473-361d-febe-1021-c7fb9d4c3b3e/testSyncNoAppend/wal.1731951345030 2024-11-18T17:35:45,163 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527)] 2024-11-18T17:35:45,173 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/d6270473-361d-febe-1021-c7fb9d4c3b3e/testSyncNoAppend 2024-11-18T17:35:45,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741871_1049 (size=93) 2024-11-18T17:35:45,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741871_1049 (size=93) 2024-11-18T17:35:45,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741871_1049 (size=93) 2024-11-18T17:35:45,603 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/d6270473-361d-febe-1021-c7fb9d4c3b3e/testSyncNoAppend 2024-11-18T17:35:45,604 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951345030) 2024-11-18T17:35:45,615 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testSyncNoAppend Thread=196 (was 197), OpenFileDescriptor=463 (was 464), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1957 (was 1770) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4108 (was 4078) - AvailableMemoryMB LEAK? - 2024-11-18T17:35:45,627 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testWriteEntryCanBeNull Thread=196, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=1957, ProcessCount=11, AvailableMemoryMB=4108 2024-11-18T17:35:45,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741872_1050 (size=7) 2024-11-18T17:35:45,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741872_1050 (size=7) 2024-11-18T17:35:45,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741872_1050 (size=7) 2024-11-18T17:35:45,716 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:45,718 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:45,720 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:45,749 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:45,749 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/88fb058f-1d0d-65ac-4da5-1eff2f2dcaf5/testWriteEntryCanBeNull, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/88fb058f-1d0d-65ac-4da5-1eff2f2dcaf5/testWriteEntryCanBeNull, maxLogs=1760 2024-11-18T17:35:45,757 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951345756 2024-11-18T17:35:45,869 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 110 ms, current pipeline: [] 2024-11-18T17:35:45,870 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/88fb058f-1d0d-65ac-4da5-1eff2f2dcaf5/testWriteEntryCanBeNull/wal.1731951345756 2024-11-18T17:35:45,870 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:39085:39085)] 2024-11-18T17:35:45,878 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/88fb058f-1d0d-65ac-4da5-1eff2f2dcaf5/testWriteEntryCanBeNull 2024-11-18T17:35:45,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741873_1051 (size=93) 2024-11-18T17:35:45,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741873_1051 (size=93) 2024-11-18T17:35:45,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741873_1051 (size=93) 2024-11-18T17:35:45,956 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/88fb058f-1d0d-65ac-4da5-1eff2f2dcaf5/testWriteEntryCanBeNull 2024-11-18T17:35:45,956 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951345756) 2024-11-18T17:35:45,976 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testWriteEntryCanBeNull Thread=196 (was 196), OpenFileDescriptor=463 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1957 (was 1957), ProcessCount=11 (was 11), AvailableMemoryMB=4077 (was 4108) 2024-11-18T17:35:46,005 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestFSHLog#testUnflushedSeqIdTrackingWithAsyncWal Thread=196, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=1957, ProcessCount=11, AvailableMemoryMB=4072 2024-11-18T17:35:46,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741874_1052 (size=7) 2024-11-18T17:35:46,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741874_1052 (size=7) 2024-11-18T17:35:46,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741874_1052 (size=7) 2024-11-18T17:35:46,132 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9 with version=8 2024-11-18T17:35:46,133 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:46,135 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T17:35:46,163 INFO [Time-limited test {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor loaded, priority=536870911. 2024-11-18T17:35:46,164 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=2 MB, rollsize=1 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testUnflushedSeqIdTrackingWithAsyncWal, archiveDir=hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs, maxLogs=1760 2024-11-18T17:35:46,169 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731951346168 2024-11-18T17:35:46,268 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testUnflushedSeqIdTrackingWithAsyncWal/wal.1731951346168 2024-11-18T17:35:46,269 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39085:39085),(127.0.0.1/127.0.0.1:37527:37527),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-18T17:35:46,270 INFO [Time-limited test {}] regionserver.HRegion(7106): creating {ENCODED => 179158d009e38d1c5f2b071b8332212d, NAME => 'table,,1731951346269.179158d009e38d1c5f2b071b8332212d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table', {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6 2024-11-18T17:35:46,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741876_1054 (size=40) 2024-11-18T17:35:46,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741876_1054 (size=40) 2024-11-18T17:35:46,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741876_1054 (size=40) 2024-11-18T17:35:46,377 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated table,,1731951346269.179158d009e38d1c5f2b071b8332212d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T17:35:46,393 INFO [StoreOpener-179158d009e38d1c5f2b071b8332212d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,400 INFO [StoreOpener-179158d009e38d1c5f2b071b8332212d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 179158d009e38d1c5f2b071b8332212d columnFamilyName b 2024-11-18T17:35:46,401 DEBUG [StoreOpener-179158d009e38d1c5f2b071b8332212d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:46,404 INFO [StoreOpener-179158d009e38d1c5f2b071b8332212d-1 {}] regionserver.HStore(327): Store=179158d009e38d1c5f2b071b8332212d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:46,406 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/table/179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,413 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/table/179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,414 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/3a201619-c066-6cc8-cc30-b154a5eb8a98/data/default/table/179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,418 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for 179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,452 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45055/user/jenkins/test-data/3a201619-c066-6cc8-cc30-b154a5eb8a98/data/default/table/179158d009e38d1c5f2b071b8332212d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T17:35:46,455 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened 179158d009e38d1c5f2b071b8332212d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71035725, jitterRate=0.0585147887468338}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T17:35:46,457 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for 179158d009e38d1c5f2b071b8332212d: 2024-11-18T17:35:46,457 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing 179158d009e38d1c5f2b071b8332212d, disabling compactions & flushes 2024-11-18T17:35:46,457 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region table,,1731951346269.179158d009e38d1c5f2b071b8332212d. 2024-11-18T17:35:46,457 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on table,,1731951346269.179158d009e38d1c5f2b071b8332212d. 2024-11-18T17:35:46,457 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on table,,1731951346269.179158d009e38d1c5f2b071b8332212d. after waiting 0 ms 2024-11-18T17:35:46,458 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region table,,1731951346269.179158d009e38d1c5f2b071b8332212d. 2024-11-18T17:35:46,464 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed table,,1731951346269.179158d009e38d1c5f2b071b8332212d. 2024-11-18T17:35:46,465 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for 179158d009e38d1c5f2b071b8332212d: 2024-11-18T17:35:46,466 DEBUG [Time-limited test {}] regionserver.HRegion(7285): Opening region: {ENCODED => 179158d009e38d1c5f2b071b8332212d, NAME => 'table,,1731951346269.179158d009e38d1c5f2b071b8332212d.', STARTKEY => '', ENDKEY => ''} 2024-11-18T17:35:46,467 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table 179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,467 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated table,,1731951346269.179158d009e38d1c5f2b071b8332212d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T17:35:46,468 DEBUG [Time-limited test {}] regionserver.HRegion(7327): checking encryption for 179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,468 DEBUG [Time-limited test {}] regionserver.HRegion(7330): checking classloading for 179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,489 INFO [StoreOpener-179158d009e38d1c5f2b071b8332212d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,497 INFO [StoreOpener-179158d009e38d1c5f2b071b8332212d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 179158d009e38d1c5f2b071b8332212d columnFamilyName b 2024-11-18T17:35:46,498 DEBUG [StoreOpener-179158d009e38d1c5f2b071b8332212d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T17:35:46,504 INFO [StoreOpener-179158d009e38d1c5f2b071b8332212d-1 {}] regionserver.HStore(327): Store=179158d009e38d1c5f2b071b8332212d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T17:35:46,508 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/table/179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,513 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/data/default/table/179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,523 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45055/user/jenkins/test-data/3a201619-c066-6cc8-cc30-b154a5eb8a98/data/default/table/179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,540 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for 179158d009e38d1c5f2b071b8332212d 2024-11-18T17:35:46,552 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened 179158d009e38d1c5f2b071b8332212d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60602966, jitterRate=-0.09694543480873108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T17:35:46,555 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for 179158d009e38d1c5f2b071b8332212d: 2024-11-18T17:35:47,198 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T17:35:49,580 DEBUG [pool-116-thread-1 {}] regionserver.HRegion(1681): Closing 179158d009e38d1c5f2b071b8332212d, disabling compactions & flushes 2024-11-18T17:35:49,580 INFO [pool-116-thread-1 {}] regionserver.HRegion(1703): Closing region table,,1731951346269.179158d009e38d1c5f2b071b8332212d. 2024-11-18T17:35:49,580 DEBUG [pool-116-thread-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on table,,1731951346269.179158d009e38d1c5f2b071b8332212d. 2024-11-18T17:35:49,580 DEBUG [pool-116-thread-1 {}] regionserver.HRegion(1791): Acquired close lock on table,,1731951346269.179158d009e38d1c5f2b071b8332212d. after waiting 0 ms 2024-11-18T17:35:49,580 DEBUG [pool-116-thread-1 {}] regionserver.HRegion(1801): Updates disabled for region table,,1731951346269.179158d009e38d1c5f2b071b8332212d. 2024-11-18T17:35:49,581 INFO [pool-116-thread-1 {}] regionserver.HRegion(2837): Flushing 179158d009e38d1c5f2b071b8332212d 1/1 column families, dataSize=48 B heapSize=448 B 2024-11-18T17:35:52,618 DEBUG [pool-116-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/179158d009e38d1c5f2b071b8332212d/.tmp/b/2a72f849ff90446d9655bb42b853fdd1 is 28, key is b/b:b/1731951346566/Put/seqid=0 2024-11-18T17:35:52,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741877_1055 (size=4945) 2024-11-18T17:35:52,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741877_1055 (size=4945) 2024-11-18T17:35:52,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741877_1055 (size=4945) 2024-11-18T17:35:52,710 INFO [pool-116-thread-1 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=6 (bloomFilter=true), to=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/179158d009e38d1c5f2b071b8332212d/.tmp/b/2a72f849ff90446d9655bb42b853fdd1 2024-11-18T17:35:52,775 DEBUG [pool-116-thread-1 {}] regionserver.HRegionFileSystem(442): Committing /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/179158d009e38d1c5f2b071b8332212d/.tmp/b/2a72f849ff90446d9655bb42b853fdd1 as /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/179158d009e38d1c5f2b071b8332212d/b/2a72f849ff90446d9655bb42b853fdd1 2024-11-18T17:35:52,818 INFO [pool-116-thread-1 {}] regionserver.HStore$StoreFlusherImpl(1989): Added /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/179158d009e38d1c5f2b071b8332212d/b/2a72f849ff90446d9655bb42b853fdd1, entries=1, sequenceid=6, filesize=4.8 K 2024-11-18T17:35:52,834 INFO [pool-116-thread-1 {}] regionserver.HRegion(3040): Finished flush of dataSize ~48 B/48, heapSize ~432 B/432, currentSize=0 B/0 for 179158d009e38d1c5f2b071b8332212d in 3253ms, sequenceid=6, compaction requested=false 2024-11-18T17:35:52,896 DEBUG [pool-116-thread-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45055/user/jenkins/test-data/3a201619-c066-6cc8-cc30-b154a5eb8a98/data/default/table/179158d009e38d1c5f2b071b8332212d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T17:35:52,901 INFO [pool-116-thread-1 {}] regionserver.HRegion(1922): Closed table,,1731951346269.179158d009e38d1c5f2b071b8332212d. 2024-11-18T17:35:52,901 DEBUG [pool-116-thread-1 {}] regionserver.HRegion(1635): Region close journal for 179158d009e38d1c5f2b071b8332212d: 2024-11-18T17:35:52,902 INFO [pool-116-thread-1 {}] wal.AbstractTestFSWAL(611): Close result:{[B@1d01c1e0=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/data/default/table/179158d009e38d1c5f2b071b8332212d/b/2a72f849ff90446d9655bb42b853fdd1]} 2024-11-18T17:35:52,902 WARN [Time-limited test {}] regionserver.HRegion(1666): Region table,,1731951346269.179158d009e38d1c5f2b071b8332212d. already closed 2024-11-18T17:35:52,902 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for 179158d009e38d1c5f2b071b8332212d: 2024-11-18T17:35:52,903 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testUnflushedSeqIdTrackingWithAsyncWal 2024-11-18T17:35:52,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39843 is added to blk_1073741875_1053 (size=1206) 2024-11-18T17:35:52,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38961 is added to blk_1073741875_1053 (size=1206) 2024-11-18T17:35:52,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46369 is added to blk_1073741875_1053 (size=1206) 2024-11-18T17:35:53,345 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/oldWALs 2024-11-18T17:35:53,345 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog wal:(num 1731951346168) 2024-11-18T17:35:53,367 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestFSHLog#testUnflushedSeqIdTrackingWithAsyncWal Thread=195 (was 196), OpenFileDescriptor=468 (was 463) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=2040 (was 1957) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3795 (was 4072) 2024-11-18T17:35:53,367 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-18T17:35:53,367 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951316585 with renewLeaseKey: DEFAULT_16417 java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK], DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK]], original=[DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK], DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DataStreamer.findNewDatanode(DataStreamer.java:1455) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.addDatanode2ExistingPipeline(DataStreamer.java:1515) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.handleDatanodeReplacement(DataStreamer.java:1758) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1648) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:53,380 WARN [PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=2:[127.0.0.1:46369, 127.0.0.1:39843] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica does not exist BP-243877644-172.17.0.3-1731951309115:1073741866 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getReplicaInfo(FsDatasetImpl.java:897) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getStorageUuidForLock(FsDatasetImpl.java:905) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.finalizeBlock(FsDatasetImpl.java:1975) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.finalizeBlock(BlockReceiver.java:1563) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1514) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:53,381 WARN [ResponseProcessor for block BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:53,382 WARN [DataStreamer for file /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344341 block BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044 {}] hdfs.DataStreamer(1731): Error Recovery for BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK], DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK], DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK]) is bad. 2024-11-18T17:35:53,394 WARN [DataStreamer for file /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344341 block BP-243877644-172.17.0.3-1731951309115:blk_1073741866_1044 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344341 (inode 16581) Holder DFSClient_NONMAPREDUCE_1869097489_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalDatanode(FSNamesystem.java:3123) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAdditionalDatanode(NameNodeRpcServer.java:963) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolServerSideTranslatorPB.java:627) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.getAdditionalDatanode(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getAdditionalDatanode$12(ClientNamenodeProtocolTranslatorPB.java:520) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolTranslatorPB.java:520) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.addDatanode2ExistingPipeline(DataStreamer.java:1505) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.handleDatanodeReplacement(DataStreamer.java:1758) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1648) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:53,397 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344341 with renewLeaseKey: DEFAULT_16581 org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344341 (inode 16581) Holder DFSClient_NONMAPREDUCE_1869097489_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalDatanode(FSNamesystem.java:3123) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAdditionalDatanode(NameNodeRpcServer.java:963) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolServerSideTranslatorPB.java:627) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.getAdditionalDatanode(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getAdditionalDatanode$12(ClientNamenodeProtocolTranslatorPB.java:520) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolTranslatorPB.java:520) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.addDatanode2ExistingPipeline(DataStreamer.java:1505) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.handleDatanodeReplacement(DataStreamer.java:1758) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1648) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:53,398 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /user/jenkins/test-data/d11f571c-e606-57da-4337-33b2fe6d67d9/testWALClosureFailureAndCleanup/wal.1731951315660 with renewLeaseKey: DEFAULT_16395 org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-243877644-172.17.0.3-1731951309115:blk_1073741826_1002 is COMPLETE but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:53,410 WARN [PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46369] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica does not exist BP-243877644-172.17.0.3-1731951309115:1073741867 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getReplicaInfo(FsDatasetImpl.java:897) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getStorageUuidForLock(FsDatasetImpl.java:905) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.finalizeBlock(FsDatasetImpl.java:1975) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.finalizeBlock(BlockReceiver.java:1563) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1514) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:53,412 WARN [ResponseProcessor for block BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045 java.io.IOException: Bad response ERROR for BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045 from datanode DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:53,412 WARN [DataStreamer for file /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344443 block BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39843,DS-869638f3-a2be-4949-87b0-7574efdf80c1,DISK], DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK], DatanodeInfoWithStorage[127.0.0.1:46369,DS-768ce038-7c52-4f14-a0f6-aca0c0fde710,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38961,DS-893706a1-b202-42fe-a55b-e134e83d4c2a,DISK]) is bad. 2024-11-18T17:35:53,412 WARN [PacketResponder: BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=2:[127.0.0.1:38961, 127.0.0.1:46369] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T17:35:53,417 WARN [DataStreamer for file /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344443 block BP-243877644-172.17.0.3-1731951309115:blk_1073741867_1045 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344443 (inode 16582) Holder DFSClient_NONMAPREDUCE_1869097489_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalDatanode(FSNamesystem.java:3123) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAdditionalDatanode(NameNodeRpcServer.java:963) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolServerSideTranslatorPB.java:627) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.getAdditionalDatanode(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getAdditionalDatanode$12(ClientNamenodeProtocolTranslatorPB.java:520) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolTranslatorPB.java:520) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.addDatanode2ExistingPipeline(DataStreamer.java:1505) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.handleDatanodeReplacement(DataStreamer.java:1758) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1648) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:53,424 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344443 with renewLeaseKey: DEFAULT_16582 org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/763ab3f7-7f56-e947-828e-5a49ee4b82d5/testFailedToCreateWALIfParentRenamed/wal.1731951344443 (inode 16582) Holder DFSClient_NONMAPREDUCE_1869097489_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalDatanode(FSNamesystem.java:3123) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAdditionalDatanode(NameNodeRpcServer.java:963) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolServerSideTranslatorPB.java:627) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.getAdditionalDatanode(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getAdditionalDatanode$12(ClientNamenodeProtocolTranslatorPB.java:520) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolTranslatorPB.java:520) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor82.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy46.getAdditionalDatanode(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.addDatanode2ExistingPipeline(DataStreamer.java:1505) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.handleDatanodeReplacement(DataStreamer.java:1758) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1648) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-18T17:35:53,474 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54d78fce{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T17:35:53,485 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71b13768{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T17:35:53,485 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T17:35:53,485 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15c9253{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T17:35:53,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4eb66c55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir/,STOPPED} 2024-11-18T17:35:53,505 WARN [BP-243877644-172.17.0.3-1731951309115 heartbeating to localhost/127.0.0.1:45055 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T17:35:53,506 WARN [BP-243877644-172.17.0.3-1731951309115 heartbeating to localhost/127.0.0.1:45055 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-243877644-172.17.0.3-1731951309115 (Datanode Uuid ddc3cc5a-81e9-4cf0-8dee-81e29bea8331) service to localhost/127.0.0.1:45055 2024-11-18T17:35:53,506 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T17:35:53,507 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T17:35:53,509 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data5/current/BP-243877644-172.17.0.3-1731951309115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T17:35:53,511 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data6/current/BP-243877644-172.17.0.3-1731951309115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T17:35:53,513 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T17:35:53,544 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d3c037d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T17:35:53,553 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30b54a1b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T17:35:53,553 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T17:35:53,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fb4c244{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T17:35:53,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19945434{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir/,STOPPED} 2024-11-18T17:35:53,585 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T17:35:53,585 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T17:35:53,586 WARN [BP-243877644-172.17.0.3-1731951309115 heartbeating to localhost/127.0.0.1:45055 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T17:35:53,586 WARN [BP-243877644-172.17.0.3-1731951309115 heartbeating to localhost/127.0.0.1:45055 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-243877644-172.17.0.3-1731951309115 (Datanode Uuid 492752f5-d29d-4b3f-9dbf-b3398a8fbed7) service to localhost/127.0.0.1:45055 2024-11-18T17:35:53,587 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data3/current/BP-243877644-172.17.0.3-1731951309115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T17:35:53,588 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data4/current/BP-243877644-172.17.0.3-1731951309115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T17:35:53,589 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T17:35:53,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b14cfe0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T17:35:53,760 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1378d4cd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T17:35:53,761 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T17:35:53,761 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5815bacf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T17:35:53,761 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ac6a421{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir/,STOPPED} 2024-11-18T17:35:53,784 WARN [BP-243877644-172.17.0.3-1731951309115 heartbeating to localhost/127.0.0.1:45055 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T17:35:53,785 WARN [BP-243877644-172.17.0.3-1731951309115 heartbeating to localhost/127.0.0.1:45055 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-243877644-172.17.0.3-1731951309115 (Datanode Uuid aa7ebd67-e5a3-4ae4-9036-63adc181b38c) service to localhost/127.0.0.1:45055 2024-11-18T17:35:53,785 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data1/current/BP-243877644-172.17.0.3-1731951309115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T17:35:53,786 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/cluster_9e3eda4f-a4fc-1296-ffb7-fa9ca07cc970/dfs/data/data2/current/BP-243877644-172.17.0.3-1731951309115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T17:35:53,788 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T17:35:53,788 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T17:35:53,800 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T17:35:53,870 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@76669e88{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T17:35:53,881 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75c47eed{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T17:35:53,881 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T17:35:53,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b6353bb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T17:35:53,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3837011d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/98f2934e-b3fa-0413-5de8-9263d0aaa5b6/hadoop.log.dir/,STOPPED} 2024-11-18T17:35:53,937 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down