2024-12-04 20:18:08,779 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-12-04 20:18:08,790 main DEBUG Took 0.008676 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 20:18:08,790 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 20:18:08,790 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 20:18:08,791 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 20:18:08,792 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,798 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 20:18:08,808 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,810 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,811 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,811 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,812 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,813 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,814 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,814 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,815 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,816 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,816 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,817 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:18:08,818 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,818 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 20:18:08,819 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:18:08,820 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 20:18:08,822 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 20:18:08,822 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 20:18:08,823 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 20:18:08,823 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 20:18:08,831 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 20:18:08,833 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 20:18:08,835 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 20:18:08,835 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 20:18:08,835 main DEBUG createAppenders(={Console}) 2024-12-04 20:18:08,836 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-12-04 20:18:08,836 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-12-04 20:18:08,836 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-12-04 20:18:08,837 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 20:18:08,837 main DEBUG OutputStream closed 2024-12-04 20:18:08,837 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 20:18:08,837 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 20:18:08,838 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-12-04 20:18:08,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 20:18:08,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 20:18:08,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 20:18:08,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 20:18:08,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 20:18:08,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 20:18:08,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 20:18:08,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 20:18:08,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 20:18:08,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 20:18:08,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 20:18:08,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 20:18:08,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 20:18:08,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 20:18:08,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 20:18:08,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 20:18:08,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 20:18:08,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 20:18:08,912 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 20:18:08,912 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-12-04 20:18:08,913 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 20:18:08,914 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-12-04T20:18:09,118 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62 2024-12-04 20:18:09,120 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 20:18:09,121 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T20:18:09,129 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-12-04T20:18:09,158 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=195, ProcessCount=11, AvailableMemoryMB=5975 2024-12-04T20:18:09,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:18:09,175 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51, deleteOnExit=true 2024-12-04T20:18:09,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:18:09,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/test.cache.data in system properties and HBase conf 2024-12-04T20:18:09,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:18:09,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:18:09,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:18:09,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:18:09,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:18:09,252 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T20:18:09,343 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:18:09,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:18:09,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:18:09,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:18:09,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:18:09,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:18:09,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:18:09,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:18:09,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:18:09,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:18:09,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:18:09,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:18:09,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:18:09,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:18:09,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:18:09,750 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:18:10,040 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T20:18:10,117 INFO [Time-limited test {}] log.Log(170): Logging initialized @1956ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T20:18:10,192 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:18:10,243 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:18:10,260 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:18:10,260 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:18:10,261 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:18:10,273 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:18:10,275 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:18:10,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:18:10,446 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/java.io.tmpdir/jetty-localhost-41991-hadoop-hdfs-3_4_1-tests_jar-_-any-6089845280733764512/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:18:10,453 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:41991} 2024-12-04T20:18:10,453 INFO [Time-limited test {}] server.Server(415): Started @2293ms 2024-12-04T20:18:10,479 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:18:10,768 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:18:10,777 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:18:10,778 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:18:10,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:18:10,779 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:18:10,780 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:18:10,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:18:10,880 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/java.io.tmpdir/jetty-localhost-41267-hadoop-hdfs-3_4_1-tests_jar-_-any-2438445741800736068/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:18:10,880 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:41267} 2024-12-04T20:18:10,881 INFO [Time-limited test {}] server.Server(415): Started @2721ms 2024-12-04T20:18:10,929 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:18:11,033 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:18:11,041 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:18:11,044 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:18:11,045 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:18:11,045 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:18:11,046 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:18:11,047 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:18:11,163 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/java.io.tmpdir/jetty-localhost-42741-hadoop-hdfs-3_4_1-tests_jar-_-any-12719141613059714879/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:18:11,164 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:42741} 2024-12-04T20:18:11,164 INFO [Time-limited test {}] server.Server(415): Started @3004ms 2024-12-04T20:18:11,167 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:18:11,286 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/data/data3/current/BP-318902724-172.17.0.2-1733343489832/current, will proceed with Du for space computation calculation, 2024-12-04T20:18:11,286 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/data/data2/current/BP-318902724-172.17.0.2-1733343489832/current, will proceed with Du for space computation calculation, 2024-12-04T20:18:11,286 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/data/data4/current/BP-318902724-172.17.0.2-1733343489832/current, will proceed with Du for space computation calculation, 2024-12-04T20:18:11,286 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/data/data1/current/BP-318902724-172.17.0.2-1733343489832/current, will proceed with Du for space computation calculation, 2024-12-04T20:18:11,345 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:18:11,345 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:18:11,409 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x172ad0e3b2676618 with lease ID 0xcc1b5aa3f45b53f3: Processing first storage report for DS-639935fb-b0dd-423c-9943-df13bba20bd2 from datanode DatanodeRegistration(127.0.0.1:46243, datanodeUuid=a238d374-c2fb-42a0-8b9b-b752432e2ab4, infoPort=41059, infoSecurePort=0, ipcPort=35361, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832) 2024-12-04T20:18:11,411 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x172ad0e3b2676618 with lease ID 0xcc1b5aa3f45b53f3: from storage DS-639935fb-b0dd-423c-9943-df13bba20bd2 node DatanodeRegistration(127.0.0.1:46243, datanodeUuid=a238d374-c2fb-42a0-8b9b-b752432e2ab4, infoPort=41059, infoSecurePort=0, ipcPort=35361, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-04T20:18:11,412 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x64298f0469c563e2 with lease ID 0xcc1b5aa3f45b53f4: Processing first storage report for DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f from datanode DatanodeRegistration(127.0.0.1:44083, datanodeUuid=715952d8-e95a-4c66-b4e6-bcc0bec57923, infoPort=45285, infoSecurePort=0, ipcPort=42175, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832) 2024-12-04T20:18:11,412 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64298f0469c563e2 with lease ID 0xcc1b5aa3f45b53f4: from storage DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f node DatanodeRegistration(127.0.0.1:44083, datanodeUuid=715952d8-e95a-4c66-b4e6-bcc0bec57923, infoPort=45285, infoSecurePort=0, ipcPort=42175, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T20:18:11,412 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x172ad0e3b2676618 with lease ID 0xcc1b5aa3f45b53f3: Processing first storage report for DS-6d5660e9-a372-472e-983d-b4546e9eb9c4 from datanode DatanodeRegistration(127.0.0.1:46243, datanodeUuid=a238d374-c2fb-42a0-8b9b-b752432e2ab4, infoPort=41059, infoSecurePort=0, ipcPort=35361, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832) 2024-12-04T20:18:11,413 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x172ad0e3b2676618 with lease ID 0xcc1b5aa3f45b53f3: from storage DS-6d5660e9-a372-472e-983d-b4546e9eb9c4 node DatanodeRegistration(127.0.0.1:46243, datanodeUuid=a238d374-c2fb-42a0-8b9b-b752432e2ab4, infoPort=41059, infoSecurePort=0, ipcPort=35361, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:18:11,413 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x64298f0469c563e2 with lease ID 0xcc1b5aa3f45b53f4: Processing first storage report for DS-a9595ec5-7060-4881-a784-4672c273223c from datanode DatanodeRegistration(127.0.0.1:44083, datanodeUuid=715952d8-e95a-4c66-b4e6-bcc0bec57923, infoPort=45285, infoSecurePort=0, ipcPort=42175, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832) 2024-12-04T20:18:11,413 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64298f0469c563e2 with lease ID 0xcc1b5aa3f45b53f4: from storage DS-a9595ec5-7060-4881-a784-4672c273223c node DatanodeRegistration(127.0.0.1:44083, datanodeUuid=715952d8-e95a-4c66-b4e6-bcc0bec57923, infoPort=45285, infoSecurePort=0, ipcPort=42175, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:18:11,505 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62 2024-12-04T20:18:11,567 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/zookeeper_0, clientPort=51308, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:18:11,576 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51308 2024-12-04T20:18:11,585 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:18:11,587 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:18:11,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:18:11,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:18:12,204 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9 with version=8 2024-12-04T20:18:12,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase-staging 2024-12-04T20:18:12,276 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T20:18:12,458 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:18:12,466 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:18:12,467 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:18:12,470 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:18:12,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:18:12,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:18:12,575 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:18:12,623 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T20:18:12,630 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T20:18:12,632 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:18:12,653 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 67247 (auto-detected) 2024-12-04T20:18:12,653 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T20:18:12,669 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37509 2024-12-04T20:18:12,686 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37509 connecting to ZooKeeper ensemble=127.0.0.1:51308 2024-12-04T20:18:12,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:375090x0, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:18:12,712 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37509-0x100a6e2124f0000 connected 2024-12-04T20:18:12,738 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:18:12,741 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:18:12,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:18:12,755 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9, hbase.cluster.distributed=false 2024-12-04T20:18:12,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:18:12,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37509 2024-12-04T20:18:12,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37509 2024-12-04T20:18:12,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37509 2024-12-04T20:18:12,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37509 2024-12-04T20:18:12,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37509 2024-12-04T20:18:12,873 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:18:12,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:18:12,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:18:12,875 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:18:12,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:18:12,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:18:12,877 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:18:12,879 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:18:12,880 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33857 2024-12-04T20:18:12,882 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33857 connecting to ZooKeeper ensemble=127.0.0.1:51308 2024-12-04T20:18:12,883 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:18:12,887 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:18:12,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338570x0, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:18:12,894 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:338570x0, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:18:12,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33857-0x100a6e2124f0001 connected 2024-12-04T20:18:12,898 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:18:12,905 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:18:12,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:18:12,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:18:12,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33857 2024-12-04T20:18:12,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33857 2024-12-04T20:18:12,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33857 2024-12-04T20:18:12,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33857 2024-12-04T20:18:12,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33857 2024-12-04T20:18:12,929 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:37509 2024-12-04T20:18:12,930 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,37509,1733343492318 2024-12-04T20:18:12,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:18:12,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:18:12,937 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,37509,1733343492318 2024-12-04T20:18:12,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:18:12,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:12,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:12,956 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:18:12,957 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,37509,1733343492318 from backup master directory 2024-12-04T20:18:12,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,37509,1733343492318 2024-12-04T20:18:12,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:18:12,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:18:12,960 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:18:12,960 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,37509,1733343492318 2024-12-04T20:18:12,962 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T20:18:12,963 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T20:18:13,016 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase.id] with ID: cd35e547-2cda-4a27-aa2a-7eb55637cd45 2024-12-04T20:18:13,016 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/.tmp/hbase.id 2024-12-04T20:18:13,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:18:13,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:18:13,030 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/.tmp/hbase.id]:[hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase.id] 2024-12-04T20:18:13,073 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:18:13,078 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:18:13,096 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-04T20:18:13,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:13,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:13,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:18:13,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:18:13,130 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:18:13,132 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:18:13,137 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:18:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:18:13,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:18:13,181 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store 2024-12-04T20:18:13,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:18:13,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:18:13,203 INFO [master/c2ef38372881:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T20:18:13,206 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:18:13,207 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:18:13,207 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:18:13,207 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:18:13,209 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:18:13,209 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:18:13,209 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:18:13,210 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343493207Disabling compacts and flushes for region at 1733343493207Disabling writes for close at 1733343493209 (+2 ms)Writing region close event to WAL at 1733343493209Closed at 1733343493209 2024-12-04T20:18:13,212 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/.initializing 2024-12-04T20:18:13,212 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/WALs/c2ef38372881,37509,1733343492318 2024-12-04T20:18:13,232 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C37509%2C1733343492318, suffix=, logDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/WALs/c2ef38372881,37509,1733343492318, archiveDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/oldWALs, maxLogs=10 2024-12-04T20:18:13,240 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37509%2C1733343492318.1733343493236 2024-12-04T20:18:13,256 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/WALs/c2ef38372881,37509,1733343492318/c2ef38372881%2C37509%2C1733343492318.1733343493236 2024-12-04T20:18:13,265 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-04T20:18:13,268 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:18:13,268 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:18:13,271 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,272 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,306 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,327 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:18:13,330 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,333 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:13,333 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:18:13,336 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:18:13,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:18:13,340 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:18:13,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,343 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:18:13,343 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:18:13,344 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,348 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,350 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,355 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,356 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,359 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:18:13,363 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:18:13,368 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:18:13,369 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799011, jitterRate=0.015995323657989502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:18:13,375 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733343493286Initializing all the Stores at 1733343493288 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343493288Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343493289 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343493289Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343493289Cleaning up temporary data from old regions at 1733343493356 (+67 ms)Region opened successfully at 1733343493375 (+19 ms) 2024-12-04T20:18:13,376 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:18:13,407 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3869540f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:18:13,433 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:18:13,442 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:18:13,442 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:18:13,445 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:18:13,446 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T20:18:13,450 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T20:18:13,451 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:18:13,476 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:18:13,484 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:18:13,486 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:18:13,488 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:18:13,489 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:18:13,490 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:18:13,492 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:18:13,495 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:18:13,496 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:18:13,498 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:18:13,499 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:18:13,513 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:18:13,514 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:18:13,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:18:13,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:18:13,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:13,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:13,520 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,37509,1733343492318, sessionid=0x100a6e2124f0000, setting cluster-up flag (Was=false) 2024-12-04T20:18:13,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:13,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:13,533 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:18:13,535 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,37509,1733343492318 2024-12-04T20:18:13,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:13,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:13,543 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:18:13,544 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,37509,1733343492318 2024-12-04T20:18:13,550 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:18:13,615 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:18:13,619 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(746): ClusterId : cd35e547-2cda-4a27-aa2a-7eb55637cd45 2024-12-04T20:18:13,621 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:18:13,623 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:18:13,625 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:18:13,625 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:18:13,628 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:18:13,629 DEBUG [RS:0;c2ef38372881:33857 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fa63213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:18:13,629 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:18:13,634 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,37509,1733343492318 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:18:13,641 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:18:13,641 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:18:13,641 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:18:13,641 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:18:13,642 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:18:13,642 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,642 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:18:13,642 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,646 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733343523646 2024-12-04T20:18:13,648 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:18:13,647 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:18:13,648 DEBUG [RS:0;c2ef38372881:33857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:33857 2024-12-04T20:18:13,648 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:18:13,649 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:18:13,652 INFO [RS:0;c2ef38372881:33857 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:18:13,652 INFO [RS:0;c2ef38372881:33857 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:18:13,652 DEBUG [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:18:13,652 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:18:13,653 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:18:13,653 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:18:13,653 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:18:13,655 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,37509,1733343492318 with port=33857, startcode=1733343492841 2024-12-04T20:18:13,655 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,655 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:18:13,654 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,657 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:18:13,658 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:18:13,659 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:18:13,663 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:18:13,663 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:18:13,665 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343493664,5,FailOnTimeoutGroup] 2024-12-04T20:18:13,666 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343493665,5,FailOnTimeoutGroup] 2024-12-04T20:18:13,666 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,666 DEBUG [RS:0;c2ef38372881:33857 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:18:13,666 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:18:13,667 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,667 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:18:13,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:18:13,676 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:18:13,676 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9 2024-12-04T20:18:13,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:18:13,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:18:13,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:18:13,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:18:13,705 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:18:13,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:13,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:18:13,713 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:18:13,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:13,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:18:13,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:18:13,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:13,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:18:13,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:18:13,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:13,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:13,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:18:13,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740 2024-12-04T20:18:13,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740 2024-12-04T20:18:13,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:18:13,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:18:13,734 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:18:13,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:18:13,741 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:18:13,741 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45853, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:18:13,742 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=719230, jitterRate=-0.08545191586017609}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:18:13,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733343493698Initializing all the Stores at 1733343493701 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343493701Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343493701Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343493701Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343493701Cleaning up temporary data from old regions at 1733343493733 (+32 ms)Region opened successfully at 1733343493745 (+12 ms) 2024-12-04T20:18:13,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:18:13,746 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:18:13,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:18:13,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:18:13,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:18:13,748 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37509 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,33857,1733343492841 2024-12-04T20:18:13,748 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:18:13,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343493746Disabling compacts and flushes for region at 1733343493746Disabling writes for close at 1733343493747 (+1 ms)Writing region close event to WAL at 1733343493748 (+1 ms)Closed at 1733343493748 2024-12-04T20:18:13,750 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37509 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,33857,1733343492841 2024-12-04T20:18:13,751 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:18:13,751 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:18:13,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:18:13,763 DEBUG [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9 2024-12-04T20:18:13,763 DEBUG [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43707 2024-12-04T20:18:13,763 DEBUG [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:18:13,764 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:18:13,767 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:18:13,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:18:13,768 DEBUG [RS:0;c2ef38372881:33857 {}] zookeeper.ZKUtil(111): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,33857,1733343492841 2024-12-04T20:18:13,768 WARN [RS:0;c2ef38372881:33857 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:18:13,768 INFO [RS:0;c2ef38372881:33857 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:18:13,769 DEBUG [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841 2024-12-04T20:18:13,771 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,33857,1733343492841] 2024-12-04T20:18:13,795 INFO [RS:0;c2ef38372881:33857 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:18:13,806 INFO [RS:0;c2ef38372881:33857 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:18:13,810 INFO [RS:0;c2ef38372881:33857 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:18:13,810 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,811 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:18:13,816 INFO [RS:0;c2ef38372881:33857 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:18:13,817 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,817 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,817 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,818 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,818 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,818 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,818 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:18:13,818 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,818 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,818 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,818 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,819 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,819 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:18:13,819 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:18:13,819 DEBUG [RS:0;c2ef38372881:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:18:13,820 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,820 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,820 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,820 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,820 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,820 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33857,1733343492841-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:18:13,836 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:18:13,837 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33857,1733343492841-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,838 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,838 INFO [RS:0;c2ef38372881:33857 {}] regionserver.Replication(171): c2ef38372881,33857,1733343492841 started 2024-12-04T20:18:13,858 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:13,859 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,33857,1733343492841, RpcServer on c2ef38372881/172.17.0.2:33857, sessionid=0x100a6e2124f0001 2024-12-04T20:18:13,860 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:18:13,860 DEBUG [RS:0;c2ef38372881:33857 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,33857,1733343492841 2024-12-04T20:18:13,860 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,33857,1733343492841' 2024-12-04T20:18:13,861 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:18:13,862 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:18:13,862 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:18:13,862 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:18:13,863 DEBUG [RS:0;c2ef38372881:33857 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,33857,1733343492841 2024-12-04T20:18:13,863 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,33857,1733343492841' 2024-12-04T20:18:13,863 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:18:13,863 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:18:13,864 DEBUG [RS:0;c2ef38372881:33857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:18:13,864 INFO [RS:0;c2ef38372881:33857 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:18:13,864 INFO [RS:0;c2ef38372881:33857 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:18:13,918 WARN [c2ef38372881:37509 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T20:18:13,977 INFO [RS:0;c2ef38372881:33857 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C33857%2C1733343492841, suffix=, logDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841, archiveDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs, maxLogs=32 2024-12-04T20:18:13,979 INFO [RS:0;c2ef38372881:33857 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343493979 2024-12-04T20:18:13,988 INFO [RS:0;c2ef38372881:33857 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343493979 2024-12-04T20:18:13,989 DEBUG [RS:0;c2ef38372881:33857 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-04T20:18:14,170 DEBUG [c2ef38372881:37509 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T20:18:14,181 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,33857,1733343492841 2024-12-04T20:18:14,188 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,33857,1733343492841, state=OPENING 2024-12-04T20:18:14,194 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:18:14,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:14,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:18:14,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:18:14,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:18:14,197 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:18:14,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,33857,1733343492841}] 2024-12-04T20:18:14,368 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:18:14,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50351, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:18:14,383 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:18:14,383 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:18:14,387 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C33857%2C1733343492841.meta, suffix=.meta, logDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841, archiveDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs, maxLogs=32 2024-12-04T20:18:14,390 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.meta.1733343494389.meta 2024-12-04T20:18:14,398 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.meta.1733343494389.meta 2024-12-04T20:18:14,399 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-04T20:18:14,400 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:18:14,402 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:18:14,404 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:18:14,408 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:18:14,411 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:18:14,412 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:18:14,412 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:18:14,412 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:18:14,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:18:14,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:18:14,417 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:14,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:14,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:18:14,419 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:18:14,419 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:14,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:14,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:18:14,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:18:14,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:14,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:14,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:18:14,425 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:18:14,425 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:14,426 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:18:14,427 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:18:14,428 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740 2024-12-04T20:18:14,431 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740 2024-12-04T20:18:14,433 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:18:14,433 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:18:14,434 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:18:14,437 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:18:14,439 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874225, jitterRate=0.11163543164730072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:18:14,439 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:18:14,440 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733343494413Writing region info on filesystem at 1733343494413Initializing all the Stores at 1733343494414 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343494415 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343494415Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343494415Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343494415Cleaning up temporary data from old regions at 1733343494433 (+18 ms)Running coprocessor post-open hooks at 1733343494439 (+6 ms)Region opened successfully at 1733343494440 (+1 ms) 2024-12-04T20:18:14,446 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733343494362 2024-12-04T20:18:14,455 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:18:14,455 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:18:14,456 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,33857,1733343492841 2024-12-04T20:18:14,458 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,33857,1733343492841, state=OPEN 2024-12-04T20:18:14,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:18:14,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:18:14,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:18:14,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:18:14,461 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,33857,1733343492841 2024-12-04T20:18:14,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:18:14,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,33857,1733343492841 in 264 msec 2024-12-04T20:18:14,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:18:14,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 712 msec 2024-12-04T20:18:14,476 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:18:14,476 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:18:14,495 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:18:14,496 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,33857,1733343492841, seqNum=-1] 2024-12-04T20:18:14,513 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:18:14,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52549, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:18:14,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 957 msec 2024-12-04T20:18:14,536 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733343494536, completionTime=-1 2024-12-04T20:18:14,538 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T20:18:14,538 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:18:14,561 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T20:18:14,561 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733343554561 2024-12-04T20:18:14,562 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733343614562 2024-12-04T20:18:14,562 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-04T20:18:14,564 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37509,1733343492318-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:14,565 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37509,1733343492318-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:14,565 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37509,1733343492318-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:14,566 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:37509, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:14,567 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:14,567 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:14,574 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:18:14,594 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.633sec 2024-12-04T20:18:14,595 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:18:14,596 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:18:14,597 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:18:14,598 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:18:14,598 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:18:14,599 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37509,1733343492318-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:18:14,599 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37509,1733343492318-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:18:14,608 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:18:14,609 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:18:14,609 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37509,1733343492318-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:18:14,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b598c24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:18:14,630 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T20:18:14,631 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T20:18:14,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,37509,-1 for getting cluster id 2024-12-04T20:18:14,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:18:14,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd35e547-2cda-4a27-aa2a-7eb55637cd45' 2024-12-04T20:18:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:18:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd35e547-2cda-4a27-aa2a-7eb55637cd45" 2024-12-04T20:18:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f22587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:18:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,37509,-1] 2024-12-04T20:18:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:18:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:18:14,655 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:18:14,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e15a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:18:14,657 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:18:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,33857,1733343492841, seqNum=-1] 2024-12-04T20:18:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:18:14,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:18:14,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,37509,1733343492318 2024-12-04T20:18:14,687 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:18:14,695 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T20:18:14,699 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:18:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,37509,1733343492318 2024-12-04T20:18:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@290bf544 2024-12-04T20:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:18:14,711 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:18:14,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37509 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T20:18:14,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37509 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T20:18:14,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37509 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:18:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37509 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-12-04T20:18:14,727 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:18:14,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37509 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-12-04T20:18:14,729 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:14,731 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:18:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:18:14,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741835_1011 (size=389) 2024-12-04T20:18:14,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741835_1011 (size=389) 2024-12-04T20:18:14,776 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0c0bde3f5329b26f5bb29bf2717aad0c, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9 2024-12-04T20:18:14,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741836_1012 (size=72) 2024-12-04T20:18:14,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741836_1012 (size=72) 2024-12-04T20:18:14,787 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:18:14,787 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 0c0bde3f5329b26f5bb29bf2717aad0c, disabling compactions & flushes 2024-12-04T20:18:14,787 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:18:14,787 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:18:14,787 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. after waiting 0 ms 2024-12-04T20:18:14,787 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:18:14,787 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:18:14,787 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0c0bde3f5329b26f5bb29bf2717aad0c: Waiting for close lock at 1733343494787Disabling compacts and flushes for region at 1733343494787Disabling writes for close at 1733343494787Writing region close event to WAL at 1733343494787Closed at 1733343494787 2024-12-04T20:18:14,789 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:18:14,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733343494790"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733343494790"}]},"ts":"1733343494790"} 2024-12-04T20:18:14,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:18:14,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:18:14,803 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343494801"}]},"ts":"1733343494801"} 2024-12-04T20:18:14,810 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-12-04T20:18:14,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0c0bde3f5329b26f5bb29bf2717aad0c, ASSIGN}] 2024-12-04T20:18:14,815 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0c0bde3f5329b26f5bb29bf2717aad0c, ASSIGN 2024-12-04T20:18:14,816 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0c0bde3f5329b26f5bb29bf2717aad0c, ASSIGN; state=OFFLINE, location=c2ef38372881,33857,1733343492841; forceNewPlan=false, retain=false 2024-12-04T20:18:14,969 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0c0bde3f5329b26f5bb29bf2717aad0c, regionState=OPENING, regionLocation=c2ef38372881,33857,1733343492841 2024-12-04T20:18:14,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0c0bde3f5329b26f5bb29bf2717aad0c, ASSIGN because future has completed 2024-12-04T20:18:14,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c0bde3f5329b26f5bb29bf2717aad0c, server=c2ef38372881,33857,1733343492841}] 2024-12-04T20:18:15,150 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:18:15,151 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c0bde3f5329b26f5bb29bf2717aad0c, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:18:15,151 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,151 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:18:15,151 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,152 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,154 INFO [StoreOpener-0c0bde3f5329b26f5bb29bf2717aad0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,157 INFO [StoreOpener-0c0bde3f5329b26f5bb29bf2717aad0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c0bde3f5329b26f5bb29bf2717aad0c columnFamilyName info 2024-12-04T20:18:15,157 DEBUG [StoreOpener-0c0bde3f5329b26f5bb29bf2717aad0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:18:15,158 INFO [StoreOpener-0c0bde3f5329b26f5bb29bf2717aad0c-1 {}] regionserver.HStore(327): Store=0c0bde3f5329b26f5bb29bf2717aad0c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:18:15,173 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,175 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,176 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,177 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,177 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,181 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,185 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:18:15,186 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0c0bde3f5329b26f5bb29bf2717aad0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=848927, jitterRate=0.07946665585041046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:18:15,186 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:15,187 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0c0bde3f5329b26f5bb29bf2717aad0c: Running coprocessor pre-open hook at 1733343495152Writing region info on filesystem at 1733343495152Initializing all the Stores at 1733343495153 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343495153Cleaning up temporary data from old regions at 1733343495177 (+24 ms)Running coprocessor post-open hooks at 1733343495186 (+9 ms)Region opened successfully at 1733343495187 (+1 ms) 2024-12-04T20:18:15,189 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c., pid=6, masterSystemTime=1733343495136 2024-12-04T20:18:15,193 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:18:15,194 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:18:15,195 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0c0bde3f5329b26f5bb29bf2717aad0c, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,33857,1733343492841 2024-12-04T20:18:15,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c0bde3f5329b26f5bb29bf2717aad0c, server=c2ef38372881,33857,1733343492841 because future has completed 2024-12-04T20:18:15,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T20:18:15,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0c0bde3f5329b26f5bb29bf2717aad0c, server=c2ef38372881,33857,1733343492841 in 226 msec 2024-12-04T20:18:15,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T20:18:15,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=0c0bde3f5329b26f5bb29bf2717aad0c, ASSIGN in 401 msec 2024-12-04T20:18:15,221 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:18:15,221 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343495221"}]},"ts":"1733343495221"} 2024-12-04T20:18:15,225 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-12-04T20:18:15,227 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:18:15,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 507 msec 2024-12-04T20:18:19,916 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:18:19,962 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T20:18:19,963 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-12-04T20:18:22,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:18:22,621 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T20:18:22,623 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-04T20:18:22,623 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T20:18:22,624 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:18:22,624 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T20:18:22,625 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T20:18:22,625 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T20:18:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:18:24,774 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-12-04T20:18:24,779 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-12-04T20:18:24,787 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-12-04T20:18:24,788 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:18:24,788 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343504788 2024-12-04T20:18:24,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:24,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:24,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:24,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:24,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:24,799 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343493979 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343504788 2024-12-04T20:18:24,800 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-04T20:18:24,800 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343493979 is not closed yet, will try archiving it next time 2024-12-04T20:18:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741833_1009 (size=451) 2024-12-04T20:18:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741833_1009 (size=451) 2024-12-04T20:18:24,804 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343493979 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs/c2ef38372881%2C33857%2C1733343492841.1733343493979 2024-12-04T20:18:24,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c., hostname=c2ef38372881,33857,1733343492841, seqNum=2] 2024-12-04T20:18:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33857 {}] regionserver.HRegion(8855): Flush requested on 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:36,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c0bde3f5329b26f5bb29bf2717aad0c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:18:36,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/7a55c09e5bb14424aec21411670800fd is 1080, key is row0001/info:/1733343504812/Put/seqid=0 2024-12-04T20:18:36,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741838_1014 (size=12509) 2024-12-04T20:18:36,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741838_1014 (size=12509) 2024-12-04T20:18:36,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/7a55c09e5bb14424aec21411670800fd 2024-12-04T20:18:36,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/7a55c09e5bb14424aec21411670800fd as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/7a55c09e5bb14424aec21411670800fd 2024-12-04T20:18:36,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/7a55c09e5bb14424aec21411670800fd, entries=7, sequenceid=11, filesize=12.2 K 2024-12-04T20:18:37,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 0c0bde3f5329b26f5bb29bf2717aad0c in 137ms, sequenceid=11, compaction requested=false 2024-12-04T20:18:37,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c0bde3f5329b26f5bb29bf2717aad0c: 2024-12-04T20:18:41,503 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:18:44,885 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343524884 2024-12-04T20:18:45,096 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 207 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK], DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK]] 2024-12-04T20:18:45,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:45,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:45,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:45,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:45,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:45,098 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343504788 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343524884 2024-12-04T20:18:45,100 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:41059:41059)] 2024-12-04T20:18:45,100 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343504788 is not closed yet, will try archiving it next time 2024-12-04T20:18:45,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741837_1013 (size=12399) 2024-12-04T20:18:45,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741837_1013 (size=12399) 2024-12-04T20:18:45,304 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:47,512 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:49,719 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:51,927 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33857 {}] regionserver.HRegion(8855): Flush requested on 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:18:51,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c0bde3f5329b26f5bb29bf2717aad0c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:18:52,132 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:52,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/ef14261cbafc4452a1f039eb57663701 is 1080, key is row0008/info:/1733343518862/Put/seqid=0 2024-12-04T20:18:52,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741840_1016 (size=12509) 2024-12-04T20:18:52,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741840_1016 (size=12509) 2024-12-04T20:18:52,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/ef14261cbafc4452a1f039eb57663701 2024-12-04T20:18:52,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/ef14261cbafc4452a1f039eb57663701 as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/ef14261cbafc4452a1f039eb57663701 2024-12-04T20:18:52,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/ef14261cbafc4452a1f039eb57663701, entries=7, sequenceid=21, filesize=12.2 K 2024-12-04T20:18:52,375 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:52,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 0c0bde3f5329b26f5bb29bf2717aad0c in 448ms, sequenceid=21, compaction requested=false 2024-12-04T20:18:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c0bde3f5329b26f5bb29bf2717aad0c: 2024-12-04T20:18:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-12-04T20:18:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:18:52,377 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/7a55c09e5bb14424aec21411670800fd because midkey is the same as first or last row 2024-12-04T20:18:54,132 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:54,995 INFO [master/c2ef38372881:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T20:18:54,995 INFO [master/c2ef38372881:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T20:18:56,338 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:56,344 WARN [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:56,346 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C33857%2C1733343492841:(num 1733343524884) roll requested 2024-12-04T20:18:56,347 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343536346 2024-12-04T20:18:56,564 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 214 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:18:56,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:56,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:56,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:56,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:56,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:18:56,566 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343524884 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343536346 2024-12-04T20:18:56,567 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-04T20:18:56,567 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343524884 is not closed yet, will try archiving it next time 2024-12-04T20:18:56,567 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343504788 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs/c2ef38372881%2C33857%2C1733343492841.1733343504788 2024-12-04T20:18:56,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741839_1015 (size=7739) 2024-12-04T20:18:56,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741839_1015 (size=7739) 2024-12-04T20:18:58,547 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK], DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK]] 2024-12-04T20:19:00,152 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 0c0bde3f5329b26f5bb29bf2717aad0c, had cached 0 bytes from a total of 25018 2024-12-04T20:19:00,754 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK], DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK]] 2024-12-04T20:19:02,962 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK], DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK]] 2024-12-04T20:19:04,385 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x172ad0e3b2676619 with lease ID 0xcc1b5aa3f45b53f5: from storage DS-639935fb-b0dd-423c-9943-df13bba20bd2 node DatanodeRegistration(127.0.0.1:46243, datanodeUuid=a238d374-c2fb-42a0-8b9b-b752432e2ab4, infoPort=41059, infoSecurePort=0, ipcPort=35361, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832), blocks: 9, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T20:19:04,385 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x172ad0e3b2676619 with lease ID 0xcc1b5aa3f45b53f5: from storage DS-6d5660e9-a372-472e-983d-b4546e9eb9c4 node DatanodeRegistration(127.0.0.1:46243, datanodeUuid=a238d374-c2fb-42a0-8b9b-b752432e2ab4, infoPort=41059, infoSecurePort=0, ipcPort=35361, storageInfo=lv=-57;cid=testClusterID;nsid=230703503;c=1733343489832), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:05,170 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK], DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK]] 2024-12-04T20:19:07,175 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T20:19:07,176 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343547175 2024-12-04T20:19:11,504 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:19:12,191 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK], DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK]] 2024-12-04T20:19:12,193 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK], DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK]] 2024-12-04T20:19:12,193 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C33857%2C1733343492841:(num 1733343547175) roll requested 2024-12-04T20:19:12,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:12,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:12,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:12,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:12,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:12,194 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343536346 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343547175 2024-12-04T20:19:12,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:41059:41059)] 2024-12-04T20:19:12,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343536346 is not closed yet, will try archiving it next time 2024-12-04T20:19:12,196 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343552195 2024-12-04T20:19:12,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741841_1017 (size=4753) 2024-12-04T20:19:12,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741841_1017 (size=4753) 2024-12-04T20:19:17,201 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:17,201 WARN [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33857 {}] regionserver.HRegion(8855): Flush requested on 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:19:17,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c0bde3f5329b26f5bb29bf2717aad0c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:19:17,212 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5012 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:17,212 WARN [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5012 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:19,203 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T20:19:22,209 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5003 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:22,210 WARN [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5003 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:22,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:22,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:22,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:22,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:22,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:22,213 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343547175 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343552195 2024-12-04T20:19:22,215 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:41059:41059)] 2024-12-04T20:19:22,215 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343547175 is not closed yet, will try archiving it next time 2024-12-04T20:19:22,216 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C33857%2C1733343492841:(num 1733343552195) roll requested 2024-12-04T20:19:22,216 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343562216 2024-12-04T20:19:22,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741842_1018 (size=1569) 2024-12-04T20:19:22,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741842_1018 (size=1569) 2024-12-04T20:19:22,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/9601704b9d8f49d4841b587f56b341f8 is 1080, key is row0015/info:/1733343533930/Put/seqid=0 2024-12-04T20:19:22,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741844_1020 (size=12509) 2024-12-04T20:19:22,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741844_1020 (size=12509) 2024-12-04T20:19:22,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/9601704b9d8f49d4841b587f56b341f8 2024-12-04T20:19:22,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/9601704b9d8f49d4841b587f56b341f8 as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/9601704b9d8f49d4841b587f56b341f8 2024-12-04T20:19:22,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/9601704b9d8f49d4841b587f56b341f8, entries=7, sequenceid=31, filesize=12.2 K 2024-12-04T20:19:27,233 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5011 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:27,233 WARN [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5011 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:27,250 INFO [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:27,250 WARN [FSHLog-0-hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9-prefix:c2ef38372881,33857,1733343492841 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44083,DS-07d72f88-7eb2-4f7e-b6cf-9ccdd6fca41f,DISK], DatanodeInfoWithStorage[127.0.0.1:46243,DS-639935fb-b0dd-423c-9943-df13bba20bd2,DISK]] 2024-12-04T20:19:27,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 0c0bde3f5329b26f5bb29bf2717aad0c in 10048ms, sequenceid=31, compaction requested=true 2024-12-04T20:19:27,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c0bde3f5329b26f5bb29bf2717aad0c: 2024-12-04T20:19:27,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,251 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-12-04T20:19:27,251 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:19:27,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,251 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/7a55c09e5bb14424aec21411670800fd because midkey is the same as first or last row 2024-12-04T20:19:27,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,253 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343552195 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343562216 2024-12-04T20:19:27,255 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:41059:41059)] 2024-12-04T20:19:27,255 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343552195 is not closed yet, will try archiving it next time 2024-12-04T20:19:27,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c0bde3f5329b26f5bb29bf2717aad0c:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:19:27,255 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343524884 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs/c2ef38372881%2C33857%2C1733343492841.1733343524884 2024-12-04T20:19:27,255 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C33857%2C1733343492841:(num 1733343562216) roll requested 2024-12-04T20:19:27,256 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343567256 2024-12-04T20:19:27,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741843_1019 (size=438) 2024-12-04T20:19:27,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741843_1019 (size=438) 2024-12-04T20:19:27,259 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343536346 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs/c2ef38372881%2C33857%2C1733343492841.1733343536346 2024-12-04T20:19:27,260 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:19:27,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:19:27,261 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343547175 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs/c2ef38372881%2C33857%2C1733343492841.1733343547175 2024-12-04T20:19:27,262 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343552195 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs/c2ef38372881%2C33857%2C1733343492841.1733343552195 2024-12-04T20:19:27,263 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:19:27,264 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.HStore(1541): 0c0bde3f5329b26f5bb29bf2717aad0c/info is initiating minor compaction (all files) 2024-12-04T20:19:27,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,265 INFO [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c0bde3f5329b26f5bb29bf2717aad0c/info in TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:19:27,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,265 INFO [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/7a55c09e5bb14424aec21411670800fd, hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/ef14261cbafc4452a1f039eb57663701, hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/9601704b9d8f49d4841b587f56b341f8] into tmpdir=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp, totalSize=36.6 K 2024-12-04T20:19:27,265 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343562216 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343567256 2024-12-04T20:19:27,267 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a55c09e5bb14424aec21411670800fd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733343504812 2024-12-04T20:19:27,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741845_1021 (size=93) 2024-12-04T20:19:27,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741845_1021 (size=93) 2024-12-04T20:19:27,268 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef14261cbafc4452a1f039eb57663701, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733343518862 2024-12-04T20:19:27,269 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9601704b9d8f49d4841b587f56b341f8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733343533930 2024-12-04T20:19:27,269 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343562216 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs/c2ef38372881%2C33857%2C1733343492841.1733343562216 2024-12-04T20:19:27,274 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-04T20:19:27,274 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33857%2C1733343492841.1733343567274 2024-12-04T20:19:27,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:27,283 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343567256 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/WALs/c2ef38372881,33857,1733343492841/c2ef38372881%2C33857%2C1733343492841.1733343567274 2024-12-04T20:19:27,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741846_1022 (size=1258) 2024-12-04T20:19:27,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741846_1022 (size=1258) 2024-12-04T20:19:27,289 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:41059:41059)] 2024-12-04T20:19:27,298 INFO [RS:0;c2ef38372881:33857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c0bde3f5329b26f5bb29bf2717aad0c#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:19:27,299 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/3a9ab808c08047cf9673a47b24cd62af is 1080, key is row0001/info:/1733343504812/Put/seqid=0 2024-12-04T20:19:27,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741848_1024 (size=27710) 2024-12-04T20:19:27,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741848_1024 (size=27710) 2024-12-04T20:19:27,315 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/3a9ab808c08047cf9673a47b24cd62af as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/3a9ab808c08047cf9673a47b24cd62af 2024-12-04T20:19:27,330 INFO [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c0bde3f5329b26f5bb29bf2717aad0c/info of 0c0bde3f5329b26f5bb29bf2717aad0c into 3a9ab808c08047cf9673a47b24cd62af(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:19:27,330 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c0bde3f5329b26f5bb29bf2717aad0c: 2024-12-04T20:19:27,332 INFO [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c., storeName=0c0bde3f5329b26f5bb29bf2717aad0c/info, priority=13, startTime=1733343567255; duration=0sec 2024-12-04T20:19:27,332 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-04T20:19:27,332 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:19:27,332 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/3a9ab808c08047cf9673a47b24cd62af because midkey is the same as first or last row 2024-12-04T20:19:27,332 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-04T20:19:27,332 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:19:27,333 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/3a9ab808c08047cf9673a47b24cd62af because midkey is the same as first or last row 2024-12-04T20:19:27,333 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-04T20:19:27,333 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:19:27,333 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/3a9ab808c08047cf9673a47b24cd62af because midkey is the same as first or last row 2024-12-04T20:19:27,333 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:19:27,333 DEBUG [RS:0;c2ef38372881:33857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c0bde3f5329b26f5bb29bf2717aad0c:info 2024-12-04T20:19:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33857 {}] regionserver.HRegion(8855): Flush requested on 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:19:39,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c0bde3f5329b26f5bb29bf2717aad0c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:19:39,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/adc690088ce246b59979ca5e769bdbac is 1080, key is row0022/info:/1733343567276/Put/seqid=0 2024-12-04T20:19:39,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741849_1025 (size=12509) 2024-12-04T20:19:39,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741849_1025 (size=12509) 2024-12-04T20:19:39,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/adc690088ce246b59979ca5e769bdbac 2024-12-04T20:19:39,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/adc690088ce246b59979ca5e769bdbac as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/adc690088ce246b59979ca5e769bdbac 2024-12-04T20:19:39,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/adc690088ce246b59979ca5e769bdbac, entries=7, sequenceid=42, filesize=12.2 K 2024-12-04T20:19:39,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 0c0bde3f5329b26f5bb29bf2717aad0c in 42ms, sequenceid=42, compaction requested=false 2024-12-04T20:19:39,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c0bde3f5329b26f5bb29bf2717aad0c: 2024-12-04T20:19:39,364 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-12-04T20:19:39,364 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:19:39,364 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/3a9ab808c08047cf9673a47b24cd62af because midkey is the same as first or last row 2024-12-04T20:19:41,504 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:19:45,153 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 0c0bde3f5329b26f5bb29bf2717aad0c, had cached 0 bytes from a total of 40219 2024-12-04T20:19:47,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:19:47,341 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:19:47,342 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:19:47,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:47,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:47,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:19:47,352 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:19:47,352 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=350154228, stopped=false 2024-12-04T20:19:47,352 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,37509,1733343492318 2024-12-04T20:19:47,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:19:47,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:19:47,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:47,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:47,353 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:19:47,353 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:19:47,354 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:19:47,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:47,354 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:19:47,354 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:19:47,354 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,33857,1733343492841' ***** 2024-12-04T20:19:47,354 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:19:47,354 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:19:47,355 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:19:47,355 INFO [RS:0;c2ef38372881:33857 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:19:47,355 INFO [RS:0;c2ef38372881:33857 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:19:47,355 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(3091): Received CLOSE for 0c0bde3f5329b26f5bb29bf2717aad0c 2024-12-04T20:19:47,355 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,33857,1733343492841 2024-12-04T20:19:47,356 INFO [RS:0;c2ef38372881:33857 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:19:47,356 INFO [RS:0;c2ef38372881:33857 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:33857. 2024-12-04T20:19:47,356 DEBUG [RS:0;c2ef38372881:33857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:19:47,356 DEBUG [RS:0;c2ef38372881:33857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:47,356 INFO [RS:0;c2ef38372881:33857 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:19:47,356 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0c0bde3f5329b26f5bb29bf2717aad0c, disabling compactions & flushes 2024-12-04T20:19:47,356 INFO [RS:0;c2ef38372881:33857 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:19:47,356 INFO [RS:0;c2ef38372881:33857 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:19:47,356 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:19:47,356 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:19:47,356 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:19:47,356 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. after waiting 0 ms 2024-12-04T20:19:47,356 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:19:47,356 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0c0bde3f5329b26f5bb29bf2717aad0c 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-12-04T20:19:47,356 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T20:19:47,356 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:19:47,356 DEBUG [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 0c0bde3f5329b26f5bb29bf2717aad0c=TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.} 2024-12-04T20:19:47,356 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:19:47,356 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:19:47,357 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:19:47,357 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:19:47,357 DEBUG [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1351): Waiting on 0c0bde3f5329b26f5bb29bf2717aad0c, 1588230740 2024-12-04T20:19:47,357 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-12-04T20:19:47,361 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/a155d02c37e242d6b2187dd6226f3685 is 1080, key is row0029/info:/1733343581324/Put/seqid=0 2024-12-04T20:19:47,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741850_1026 (size=8193) 2024-12-04T20:19:47,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741850_1026 (size=8193) 2024-12-04T20:19:47,369 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/a155d02c37e242d6b2187dd6226f3685 2024-12-04T20:19:47,378 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/.tmp/info/a155d02c37e242d6b2187dd6226f3685 as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/a155d02c37e242d6b2187dd6226f3685 2024-12-04T20:19:47,378 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/info/41c02b96630943d18cf594fb8bccbd34 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c./info:regioninfo/1733343495195/Put/seqid=0 2024-12-04T20:19:47,386 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/a155d02c37e242d6b2187dd6226f3685, entries=3, sequenceid=48, filesize=8.0 K 2024-12-04T20:19:47,387 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 0c0bde3f5329b26f5bb29bf2717aad0c in 31ms, sequenceid=48, compaction requested=true 2024-12-04T20:19:47,393 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/7a55c09e5bb14424aec21411670800fd, hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/ef14261cbafc4452a1f039eb57663701, hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/9601704b9d8f49d4841b587f56b341f8] to archive 2024-12-04T20:19:47,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741851_1027 (size=7016) 2024-12-04T20:19:47,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741851_1027 (size=7016) 2024-12-04T20:19:47,396 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/info/41c02b96630943d18cf594fb8bccbd34 2024-12-04T20:19:47,397 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:19:47,400 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/7a55c09e5bb14424aec21411670800fd to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/archive/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/7a55c09e5bb14424aec21411670800fd 2024-12-04T20:19:47,403 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/ef14261cbafc4452a1f039eb57663701 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/archive/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/ef14261cbafc4452a1f039eb57663701 2024-12-04T20:19:47,405 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/9601704b9d8f49d4841b587f56b341f8 to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/archive/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/info/9601704b9d8f49d4841b587f56b341f8 2024-12-04T20:19:47,418 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/ns/1cd38afb9d4144918c9dc71d3e76ab04 is 43, key is default/ns:d/1733343494518/Put/seqid=0 2024-12-04T20:19:47,415 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c2ef38372881:37509 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T20:19:47,418 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [7a55c09e5bb14424aec21411670800fd=12509, ef14261cbafc4452a1f039eb57663701=12509, 9601704b9d8f49d4841b587f56b341f8=12509] 2024-12-04T20:19:47,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741852_1028 (size=5153) 2024-12-04T20:19:47,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741852_1028 (size=5153) 2024-12-04T20:19:47,424 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/ns/1cd38afb9d4144918c9dc71d3e76ab04 2024-12-04T20:19:47,424 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/default/TestLogRolling-testSlowSyncLogRolling/0c0bde3f5329b26f5bb29bf2717aad0c/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-12-04T20:19:47,427 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:19:47,427 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0c0bde3f5329b26f5bb29bf2717aad0c: Waiting for close lock at 1733343587356Running coprocessor pre-close hooks at 1733343587356Disabling compacts and flushes for region at 1733343587356Disabling writes for close at 1733343587356Obtaining lock to block concurrent updates at 1733343587356Preparing flush snapshotting stores in 0c0bde3f5329b26f5bb29bf2717aad0c at 1733343587356Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733343587357 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. at 1733343587357Flushing 0c0bde3f5329b26f5bb29bf2717aad0c/info: creating writer at 1733343587357Flushing 0c0bde3f5329b26f5bb29bf2717aad0c/info: appending metadata at 1733343587361 (+4 ms)Flushing 0c0bde3f5329b26f5bb29bf2717aad0c/info: closing flushed file at 1733343587361Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b20b96e: reopening flushed file at 1733343587377 (+16 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 0c0bde3f5329b26f5bb29bf2717aad0c in 31ms, sequenceid=48, compaction requested=true at 1733343587387 (+10 ms)Writing region close event to WAL at 1733343587419 (+32 ms)Running coprocessor post-close hooks at 1733343587425 (+6 ms)Closed at 1733343587427 (+2 ms) 2024-12-04T20:19:47,428 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733343494713.0c0bde3f5329b26f5bb29bf2717aad0c. 2024-12-04T20:19:47,445 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/table/79d5477571fe45abb32868b1ddc0091e is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733343495221/Put/seqid=0 2024-12-04T20:19:47,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741853_1029 (size=5396) 2024-12-04T20:19:47,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741853_1029 (size=5396) 2024-12-04T20:19:47,451 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/table/79d5477571fe45abb32868b1ddc0091e 2024-12-04T20:19:47,460 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/info/41c02b96630943d18cf594fb8bccbd34 as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/info/41c02b96630943d18cf594fb8bccbd34 2024-12-04T20:19:47,468 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/info/41c02b96630943d18cf594fb8bccbd34, entries=10, sequenceid=11, filesize=6.9 K 2024-12-04T20:19:47,470 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/ns/1cd38afb9d4144918c9dc71d3e76ab04 as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/ns/1cd38afb9d4144918c9dc71d3e76ab04 2024-12-04T20:19:47,479 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/ns/1cd38afb9d4144918c9dc71d3e76ab04, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T20:19:47,480 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/.tmp/table/79d5477571fe45abb32868b1ddc0091e as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/table/79d5477571fe45abb32868b1ddc0091e 2024-12-04T20:19:47,488 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/table/79d5477571fe45abb32868b1ddc0091e, entries=2, sequenceid=11, filesize=5.3 K 2024-12-04T20:19:47,490 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 133ms, sequenceid=11, compaction requested=false 2024-12-04T20:19:47,496 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T20:19:47,497 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:19:47,498 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:19:47,498 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343587356Running coprocessor pre-close hooks at 1733343587356Disabling compacts and flushes for region at 1733343587356Disabling writes for close at 1733343587357 (+1 ms)Obtaining lock to block concurrent updates at 1733343587357Preparing flush snapshotting stores in 1588230740 at 1733343587357Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733343587357Flushing stores of hbase:meta,,1.1588230740 at 1733343587358 (+1 ms)Flushing 1588230740/info: creating writer at 1733343587358Flushing 1588230740/info: appending metadata at 1733343587378 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733343587378Flushing 1588230740/ns: creating writer at 1733343587403 (+25 ms)Flushing 1588230740/ns: appending metadata at 1733343587417 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733343587417Flushing 1588230740/table: creating writer at 1733343587431 (+14 ms)Flushing 1588230740/table: appending metadata at 1733343587444 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733343587445 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39b5e675: reopening flushed file at 1733343587459 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4768c9d9: reopening flushed file at 1733343587469 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61628c86: reopening flushed file at 1733343587479 (+10 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 133ms, sequenceid=11, compaction requested=false at 1733343587490 (+11 ms)Writing region close event to WAL at 1733343587492 (+2 ms)Running coprocessor post-close hooks at 1733343587497 (+5 ms)Closed at 1733343587498 (+1 ms) 2024-12-04T20:19:47,498 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:19:47,557 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,33857,1733343492841; all regions closed. 2024-12-04T20:19:47,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741834_1010 (size=3066) 2024-12-04T20:19:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741834_1010 (size=3066) 2024-12-04T20:19:47,571 DEBUG [RS:0;c2ef38372881:33857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs 2024-12-04T20:19:47,571 INFO [RS:0;c2ef38372881:33857 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C33857%2C1733343492841.meta:.meta(num 1733343494389) 2024-12-04T20:19:47,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741847_1023 (size=12695) 2024-12-04T20:19:47,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741847_1023 (size=12695) 2024-12-04T20:19:47,578 DEBUG [RS:0;c2ef38372881:33857 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/oldWALs 2024-12-04T20:19:47,578 INFO [RS:0;c2ef38372881:33857 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C33857%2C1733343492841:(num 1733343567274) 2024-12-04T20:19:47,578 DEBUG [RS:0;c2ef38372881:33857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:47,578 INFO [RS:0;c2ef38372881:33857 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:19:47,579 INFO [RS:0;c2ef38372881:33857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:19:47,579 INFO [RS:0;c2ef38372881:33857 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T20:19:47,579 INFO [RS:0;c2ef38372881:33857 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:19:47,579 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:19:47,579 INFO [RS:0;c2ef38372881:33857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33857 2024-12-04T20:19:47,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:19:47,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,33857,1733343492841 2024-12-04T20:19:47,583 INFO [RS:0;c2ef38372881:33857 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:19:47,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,33857,1733343492841] 2024-12-04T20:19:47,585 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,33857,1733343492841 already deleted, retry=false 2024-12-04T20:19:47,585 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,33857,1733343492841 expired; onlineServers=0 2024-12-04T20:19:47,585 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,37509,1733343492318' ***** 2024-12-04T20:19:47,585 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:19:47,585 INFO [M:0;c2ef38372881:37509 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:19:47,586 INFO [M:0;c2ef38372881:37509 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:19:47,586 DEBUG [M:0;c2ef38372881:37509 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:19:47,586 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:19:47,586 DEBUG [M:0;c2ef38372881:37509 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:19:47,586 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343493664 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343493664,5,FailOnTimeoutGroup] 2024-12-04T20:19:47,586 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343493665 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343493665,5,FailOnTimeoutGroup] 2024-12-04T20:19:47,586 INFO [M:0;c2ef38372881:37509 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:19:47,586 INFO [M:0;c2ef38372881:37509 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:19:47,586 DEBUG [M:0;c2ef38372881:37509 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:19:47,586 INFO [M:0;c2ef38372881:37509 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:19:47,586 INFO [M:0;c2ef38372881:37509 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:19:47,587 INFO [M:0;c2ef38372881:37509 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:19:47,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:19:47,587 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:19:47,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:47,587 DEBUG [M:0;c2ef38372881:37509 {}] zookeeper.ZKUtil(347): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:19:47,587 WARN [M:0;c2ef38372881:37509 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:19:47,588 INFO [M:0;c2ef38372881:37509 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/.lastflushedseqids 2024-12-04T20:19:47,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741854_1030 (size=130) 2024-12-04T20:19:47,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741854_1030 (size=130) 2024-12-04T20:19:47,599 INFO [M:0;c2ef38372881:37509 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:19:47,600 INFO [M:0;c2ef38372881:37509 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:19:47,600 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:19:47,600 INFO [M:0;c2ef38372881:37509 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:47,600 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:47,600 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:19:47,600 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:47,600 INFO [M:0;c2ef38372881:37509 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB 2024-12-04T20:19:47,620 DEBUG [M:0;c2ef38372881:37509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2ce704ff8cb84ca7b54ad7c6fc415037 is 82, key is hbase:meta,,1/info:regioninfo/1733343494456/Put/seqid=0 2024-12-04T20:19:47,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741855_1031 (size=5672) 2024-12-04T20:19:47,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741855_1031 (size=5672) 2024-12-04T20:19:47,626 INFO [M:0;c2ef38372881:37509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2ce704ff8cb84ca7b54ad7c6fc415037 2024-12-04T20:19:47,649 DEBUG [M:0;c2ef38372881:37509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b844db78b0a41bc9183aab840258b27 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733343495229/Put/seqid=0 2024-12-04T20:19:47,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741856_1032 (size=6247) 2024-12-04T20:19:47,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741856_1032 (size=6247) 2024-12-04T20:19:47,656 INFO [M:0;c2ef38372881:37509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b844db78b0a41bc9183aab840258b27 2024-12-04T20:19:47,661 INFO [M:0;c2ef38372881:37509 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5b844db78b0a41bc9183aab840258b27 2024-12-04T20:19:47,676 DEBUG [M:0;c2ef38372881:37509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9bfd6846edfe4aeab64356a3a18688b9 is 69, key is c2ef38372881,33857,1733343492841/rs:state/1733343493752/Put/seqid=0 2024-12-04T20:19:47,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741857_1033 (size=5156) 2024-12-04T20:19:47,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741857_1033 (size=5156) 2024-12-04T20:19:47,682 INFO [M:0;c2ef38372881:37509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9bfd6846edfe4aeab64356a3a18688b9 2024-12-04T20:19:47,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:19:47,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x100a6e2124f0001, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:19:47,685 INFO [RS:0;c2ef38372881:33857 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:19:47,685 INFO [RS:0;c2ef38372881:33857 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,33857,1733343492841; zookeeper connection closed. 2024-12-04T20:19:47,685 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7952bf2d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7952bf2d 2024-12-04T20:19:47,686 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T20:19:47,702 DEBUG [M:0;c2ef38372881:37509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/48c637d0f13442c293f5ee39ff71786c is 52, key is load_balancer_on/state:d/1733343494691/Put/seqid=0 2024-12-04T20:19:47,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741858_1034 (size=5056) 2024-12-04T20:19:47,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741858_1034 (size=5056) 2024-12-04T20:19:47,709 INFO [M:0;c2ef38372881:37509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/48c637d0f13442c293f5ee39ff71786c 2024-12-04T20:19:47,717 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2ce704ff8cb84ca7b54ad7c6fc415037 as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2ce704ff8cb84ca7b54ad7c6fc415037 2024-12-04T20:19:47,724 INFO [M:0;c2ef38372881:37509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2ce704ff8cb84ca7b54ad7c6fc415037, entries=8, sequenceid=59, filesize=5.5 K 2024-12-04T20:19:47,725 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b844db78b0a41bc9183aab840258b27 as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5b844db78b0a41bc9183aab840258b27 2024-12-04T20:19:47,734 INFO [M:0;c2ef38372881:37509 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5b844db78b0a41bc9183aab840258b27 2024-12-04T20:19:47,734 INFO [M:0;c2ef38372881:37509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5b844db78b0a41bc9183aab840258b27, entries=6, sequenceid=59, filesize=6.1 K 2024-12-04T20:19:47,736 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9bfd6846edfe4aeab64356a3a18688b9 as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9bfd6846edfe4aeab64356a3a18688b9 2024-12-04T20:19:47,744 INFO [M:0;c2ef38372881:37509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9bfd6846edfe4aeab64356a3a18688b9, entries=1, sequenceid=59, filesize=5.0 K 2024-12-04T20:19:47,745 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/48c637d0f13442c293f5ee39ff71786c as hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/48c637d0f13442c293f5ee39ff71786c 2024-12-04T20:19:47,753 INFO [M:0;c2ef38372881:37509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/48c637d0f13442c293f5ee39ff71786c, entries=1, sequenceid=59, filesize=4.9 K 2024-12-04T20:19:47,754 INFO [M:0;c2ef38372881:37509 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 154ms, sequenceid=59, compaction requested=false 2024-12-04T20:19:47,756 INFO [M:0;c2ef38372881:37509 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:47,756 DEBUG [M:0;c2ef38372881:37509 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343587600Disabling compacts and flushes for region at 1733343587600Disabling writes for close at 1733343587600Obtaining lock to block concurrent updates at 1733343587600Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733343587600Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1733343587601 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733343587602 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733343587602Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733343587620 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733343587620Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733343587633 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733343587648 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733343587648Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733343587661 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733343587675 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733343587676 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733343587689 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733343587702 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733343587702Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e27d65e: reopening flushed file at 1733343587716 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@91726a0: reopening flushed file at 1733343587724 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@433c209f: reopening flushed file at 1733343587735 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a8362f8: reopening flushed file at 1733343587744 (+9 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 154ms, sequenceid=59, compaction requested=false at 1733343587754 (+10 ms)Writing region close event to WAL at 1733343587756 (+2 ms)Closed at 1733343587756 2024-12-04T20:19:47,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:47,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44083 is added to blk_1073741830_1006 (size=27973) 2024-12-04T20:19:47,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46243 is added to blk_1073741830_1006 (size=27973) 2024-12-04T20:19:47,760 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:19:47,760 INFO [M:0;c2ef38372881:37509 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:19:47,761 INFO [M:0;c2ef38372881:37509 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37509 2024-12-04T20:19:47,761 INFO [M:0;c2ef38372881:37509 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:19:47,827 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:19:47,862 INFO [M:0;c2ef38372881:37509 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:19:47,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:19:47,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37509-0x100a6e2124f0000, quorum=127.0.0.1:51308, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:19:47,871 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:19:47,874 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:19:47,874 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:19:47,874 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:19:47,874 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.log.dir/,STOPPED} 2024-12-04T20:19:47,877 WARN [BP-318902724-172.17.0.2-1733343489832 heartbeating to localhost/127.0.0.1:43707 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:19:47,877 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:19:47,877 WARN [BP-318902724-172.17.0.2-1733343489832 heartbeating to localhost/127.0.0.1:43707 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-318902724-172.17.0.2-1733343489832 (Datanode Uuid 715952d8-e95a-4c66-b4e6-bcc0bec57923) service to localhost/127.0.0.1:43707 2024-12-04T20:19:47,877 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:19:47,878 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/data/data3/current/BP-318902724-172.17.0.2-1733343489832 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:19:47,878 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/data/data4/current/BP-318902724-172.17.0.2-1733343489832 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:19:47,879 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:19:47,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:19:47,881 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:19:47,881 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:19:47,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:19:47,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.log.dir/,STOPPED} 2024-12-04T20:19:47,883 WARN [BP-318902724-172.17.0.2-1733343489832 heartbeating to localhost/127.0.0.1:43707 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:19:47,883 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:19:47,883 WARN [BP-318902724-172.17.0.2-1733343489832 heartbeating to localhost/127.0.0.1:43707 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-318902724-172.17.0.2-1733343489832 (Datanode Uuid a238d374-c2fb-42a0-8b9b-b752432e2ab4) service to localhost/127.0.0.1:43707 2024-12-04T20:19:47,883 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:19:47,884 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/data/data1/current/BP-318902724-172.17.0.2-1733343489832 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:19:47,884 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/cluster_73394149-ecbc-46ee-6054-bd4871ceae51/data/data2/current/BP-318902724-172.17.0.2-1733343489832 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:19:47,884 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:19:47,891 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:19:47,891 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:19:47,891 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:19:47,892 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:19:47,892 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.log.dir/,STOPPED} 2024-12-04T20:19:47,900 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:19:47,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T20:19:47,942 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=81 (was 12) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43707 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43707 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43707 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/c2ef38372881:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43707 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: master/c2ef38372881:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool-2-worker-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c2ef38372881:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@1d316e22 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43707 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43707 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:43707 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:43707 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=52 (was 195), ProcessCount=11 (was 11), AvailableMemoryMB=5382 (was 5975) 2024-12-04T20:19:47,948 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=82, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=52, ProcessCount=11, AvailableMemoryMB=5380 2024-12-04T20:19:47,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:19:47,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.log.dir so I do NOT create it in target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4 2024-12-04T20:19:47,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce7c8869-4ec5-1c70-39b0-0e789e351b62/hadoop.tmp.dir so I do NOT create it in target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4 2024-12-04T20:19:47,949 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d, deleteOnExit=true 2024-12-04T20:19:47,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:19:47,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/test.cache.data in system properties and HBase conf 2024-12-04T20:19:47,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:19:47,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:19:47,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:19:47,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:19:47,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:19:47,949 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:19:47,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:19:47,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:19:47,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:19:47,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:19:47,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:19:47,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:19:47,965 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:19:48,015 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:19:48,020 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:19:48,021 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:19:48,021 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:19:48,022 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:19:48,022 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:19:48,023 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f681677{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:19:48,023 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3197ca45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:19:48,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49a88a00{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/java.io.tmpdir/jetty-localhost-45237-hadoop-hdfs-3_4_1-tests_jar-_-any-74847751547955681/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:19:48,115 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e4256d4{HTTP/1.1, (http/1.1)}{localhost:45237} 2024-12-04T20:19:48,115 INFO [Time-limited test {}] server.Server(415): Started @99955ms 2024-12-04T20:19:48,127 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:19:48,174 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:19:48,178 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:19:48,179 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:19:48,179 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:19:48,179 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:19:48,179 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@eab7acc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:19:48,180 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4edee9ab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:19:48,269 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@542ee468{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/java.io.tmpdir/jetty-localhost-33921-hadoop-hdfs-3_4_1-tests_jar-_-any-1857337361885327030/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:19:48,270 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75c88313{HTTP/1.1, (http/1.1)}{localhost:33921} 2024-12-04T20:19:48,270 INFO [Time-limited test {}] server.Server(415): Started @100110ms 2024-12-04T20:19:48,271 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:19:48,303 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:19:48,309 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:19:48,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:19:48,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:19:48,309 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:19:48,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c1be80f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:19:48,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b44e274{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:19:48,337 WARN [Thread-439 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/data/data2/current/BP-971878257-172.17.0.2-1733343587976/current, will proceed with Du for space computation calculation, 2024-12-04T20:19:48,337 WARN [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/data/data1/current/BP-971878257-172.17.0.2-1733343587976/current, will proceed with Du for space computation calculation, 2024-12-04T20:19:48,351 WARN [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:19:48,354 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf00f9e53aff0b5aa with lease ID 0x2bdf6a0a18106caf: Processing first storage report for DS-8d9e1c5c-8b46-43ce-ac08-671eee92d4dd from datanode DatanodeRegistration(127.0.0.1:44465, datanodeUuid=35a876a8-11e7-4cc9-b67a-7d0c5fcdca52, infoPort=41701, infoSecurePort=0, ipcPort=33289, storageInfo=lv=-57;cid=testClusterID;nsid=936232817;c=1733343587976) 2024-12-04T20:19:48,354 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf00f9e53aff0b5aa with lease ID 0x2bdf6a0a18106caf: from storage DS-8d9e1c5c-8b46-43ce-ac08-671eee92d4dd node DatanodeRegistration(127.0.0.1:44465, datanodeUuid=35a876a8-11e7-4cc9-b67a-7d0c5fcdca52, infoPort=41701, infoSecurePort=0, ipcPort=33289, storageInfo=lv=-57;cid=testClusterID;nsid=936232817;c=1733343587976), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:48,354 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf00f9e53aff0b5aa with lease ID 0x2bdf6a0a18106caf: Processing first storage report for DS-ccd4ee90-3075-44f1-864f-b240b8c6888a from datanode DatanodeRegistration(127.0.0.1:44465, datanodeUuid=35a876a8-11e7-4cc9-b67a-7d0c5fcdca52, infoPort=41701, infoSecurePort=0, ipcPort=33289, storageInfo=lv=-57;cid=testClusterID;nsid=936232817;c=1733343587976) 2024-12-04T20:19:48,354 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf00f9e53aff0b5aa with lease ID 0x2bdf6a0a18106caf: from storage DS-ccd4ee90-3075-44f1-864f-b240b8c6888a node DatanodeRegistration(127.0.0.1:44465, datanodeUuid=35a876a8-11e7-4cc9-b67a-7d0c5fcdca52, infoPort=41701, infoSecurePort=0, ipcPort=33289, storageInfo=lv=-57;cid=testClusterID;nsid=936232817;c=1733343587976), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:48,409 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d183c93{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/java.io.tmpdir/jetty-localhost-34295-hadoop-hdfs-3_4_1-tests_jar-_-any-3616727790539980514/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:19:48,409 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7154ca22{HTTP/1.1, (http/1.1)}{localhost:34295} 2024-12-04T20:19:48,409 INFO [Time-limited test {}] server.Server(415): Started @100249ms 2024-12-04T20:19:48,410 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:19:48,466 WARN [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/data/data3/current/BP-971878257-172.17.0.2-1733343587976/current, will proceed with Du for space computation calculation, 2024-12-04T20:19:48,466 WARN [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/data/data4/current/BP-971878257-172.17.0.2-1733343587976/current, will proceed with Du for space computation calculation, 2024-12-04T20:19:48,485 WARN [Thread-453 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:19:48,487 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9b5527d13e40e1d with lease ID 0x2bdf6a0a18106cb0: Processing first storage report for DS-33a7911e-1bf1-4525-b96f-0c9d64491990 from datanode DatanodeRegistration(127.0.0.1:41875, datanodeUuid=b6e8ce5b-c0e8-455a-b76f-e338c14b1d7b, infoPort=36427, infoSecurePort=0, ipcPort=43957, storageInfo=lv=-57;cid=testClusterID;nsid=936232817;c=1733343587976) 2024-12-04T20:19:48,487 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9b5527d13e40e1d with lease ID 0x2bdf6a0a18106cb0: from storage DS-33a7911e-1bf1-4525-b96f-0c9d64491990 node DatanodeRegistration(127.0.0.1:41875, datanodeUuid=b6e8ce5b-c0e8-455a-b76f-e338c14b1d7b, infoPort=36427, infoSecurePort=0, ipcPort=43957, storageInfo=lv=-57;cid=testClusterID;nsid=936232817;c=1733343587976), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:48,487 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9b5527d13e40e1d with lease ID 0x2bdf6a0a18106cb0: Processing first storage report for DS-b2baca0a-ccd3-4d3d-81af-1aad5ccf50aa from datanode DatanodeRegistration(127.0.0.1:41875, datanodeUuid=b6e8ce5b-c0e8-455a-b76f-e338c14b1d7b, infoPort=36427, infoSecurePort=0, ipcPort=43957, storageInfo=lv=-57;cid=testClusterID;nsid=936232817;c=1733343587976) 2024-12-04T20:19:48,487 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9b5527d13e40e1d with lease ID 0x2bdf6a0a18106cb0: from storage DS-b2baca0a-ccd3-4d3d-81af-1aad5ccf50aa node DatanodeRegistration(127.0.0.1:41875, datanodeUuid=b6e8ce5b-c0e8-455a-b76f-e338c14b1d7b, infoPort=36427, infoSecurePort=0, ipcPort=43957, storageInfo=lv=-57;cid=testClusterID;nsid=936232817;c=1733343587976), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:48,535 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4 2024-12-04T20:19:48,538 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/zookeeper_0, clientPort=58516, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:19:48,539 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58516 2024-12-04T20:19:48,539 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:48,541 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:48,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:19:48,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:19:48,551 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367 with version=8 2024-12-04T20:19:48,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase-staging 2024-12-04T20:19:48,553 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:19:48,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:48,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:48,554 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:19:48,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:48,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:19:48,554 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:19:48,554 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:19:48,558 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44137 2024-12-04T20:19:48,559 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44137 connecting to ZooKeeper ensemble=127.0.0.1:58516 2024-12-04T20:19:48,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:441370x0, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:19:48,563 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44137-0x100a6e38d280000 connected 2024-12-04T20:19:48,573 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:48,574 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:48,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:19:48,577 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367, hbase.cluster.distributed=false 2024-12-04T20:19:48,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:19:48,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44137 2024-12-04T20:19:48,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44137 2024-12-04T20:19:48,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44137 2024-12-04T20:19:48,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44137 2024-12-04T20:19:48,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44137 2024-12-04T20:19:48,596 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:19:48,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:48,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:48,597 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:19:48,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:48,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:19:48,597 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:19:48,597 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:19:48,598 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37995 2024-12-04T20:19:48,599 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37995 connecting to ZooKeeper ensemble=127.0.0.1:58516 2024-12-04T20:19:48,599 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:48,601 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:48,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379950x0, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:19:48,606 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:379950x0, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:19:48,606 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:19:48,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37995-0x100a6e38d280001 connected 2024-12-04T20:19:48,608 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:19:48,609 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:19:48,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:19:48,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-04T20:19:48,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37995 2024-12-04T20:19:48,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37995 2024-12-04T20:19:48,618 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-04T20:19:48,618 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-04T20:19:48,629 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:44137 2024-12-04T20:19:48,630 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,44137,1733343588553 2024-12-04T20:19:48,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:19:48,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:19:48,632 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,44137,1733343588553 2024-12-04T20:19:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:19:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:48,633 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:19:48,634 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,44137,1733343588553 from backup master directory 2024-12-04T20:19:48,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,44137,1733343588553 2024-12-04T20:19:48,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:19:48,635 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:19:48,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:19:48,635 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,44137,1733343588553 2024-12-04T20:19:48,642 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/hbase.id] with ID: 9d0572e5-d9a1-4224-aef9-01c3d6dfc0be 2024-12-04T20:19:48,642 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/.tmp/hbase.id 2024-12-04T20:19:48,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:19:48,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:19:48,651 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/.tmp/hbase.id]:[hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/hbase.id] 2024-12-04T20:19:48,668 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:48,669 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:19:48,670 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T20:19:48,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:48,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:48,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:19:48,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:19:48,681 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:19:48,682 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:19:48,683 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:19:48,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:19:48,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:19:49,097 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store 2024-12-04T20:19:49,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:19:49,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:19:49,107 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:49,107 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:19:49,107 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:49,107 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:49,107 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:19:49,107 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:49,107 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:49,107 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343589107Disabling compacts and flushes for region at 1733343589107Disabling writes for close at 1733343589107Writing region close event to WAL at 1733343589107Closed at 1733343589107 2024-12-04T20:19:49,109 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/.initializing 2024-12-04T20:19:49,109 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/WALs/c2ef38372881,44137,1733343588553 2024-12-04T20:19:49,112 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C44137%2C1733343588553, suffix=, logDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/WALs/c2ef38372881,44137,1733343588553, archiveDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/oldWALs, maxLogs=10 2024-12-04T20:19:49,113 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C44137%2C1733343588553.1733343589113 2024-12-04T20:19:49,119 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/WALs/c2ef38372881,44137,1733343588553/c2ef38372881%2C44137%2C1733343588553.1733343589113 2024-12-04T20:19:49,120 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36427:36427),(127.0.0.1/127.0.0.1:41701:41701)] 2024-12-04T20:19:49,121 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:19:49,121 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:49,121 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,121 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:19:49,127 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:19:49,129 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:19:49,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,132 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:19:49,133 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:19:49,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:19:49,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:19:49,136 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,138 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,138 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,140 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,140 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,141 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:19:49,143 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:49,145 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:19:49,146 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=848151, jitterRate=0.07847991585731506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:19:49,147 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733343589121Initializing all the Stores at 1733343589122 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343589122Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343589125 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343589125Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343589125Cleaning up temporary data from old regions at 1733343589140 (+15 ms)Region opened successfully at 1733343589147 (+7 ms) 2024-12-04T20:19:49,147 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:19:49,152 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c8fa26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:19:49,154 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:19:49,154 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:19:49,154 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:19:49,154 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:19:49,155 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T20:19:49,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T20:19:49,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:19:49,159 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:19:49,160 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:19:49,161 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:19:49,162 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:19:49,163 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:19:49,163 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:19:49,164 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:19:49,166 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:19:49,167 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:19:49,168 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:19:49,168 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:19:49,170 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:19:49,171 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:19:49,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:19:49,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:19:49,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,173 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,44137,1733343588553, sessionid=0x100a6e38d280000, setting cluster-up flag (Was=false) 2024-12-04T20:19:49,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,178 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:19:49,179 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,44137,1733343588553 2024-12-04T20:19:49,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,184 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:19:49,185 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,44137,1733343588553 2024-12-04T20:19:49,187 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:19:49,189 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:19:49,189 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:19:49,189 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:19:49,189 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,44137,1733343588553 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:19:49,191 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:19:49,191 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:19:49,191 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:19:49,191 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:19:49,191 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:19:49,192 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,192 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:19:49,192 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,192 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733343619192 2024-12-04T20:19:49,193 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:19:49,193 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:19:49,193 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:19:49,193 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:19:49,193 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:19:49,193 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:19:49,194 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:19:49,194 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:19:49,195 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,195 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:19:49,197 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,197 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:19:49,197 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:19:49,197 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:19:49,197 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:19:49,197 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:19:49,198 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343589198,5,FailOnTimeoutGroup] 2024-12-04T20:19:49,198 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343589198,5,FailOnTimeoutGroup] 2024-12-04T20:19:49,198 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,198 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:19:49,198 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,198 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:19:49,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:19:49,204 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:19:49,204 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367 2024-12-04T20:19:49,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:19:49,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:19:49,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:49,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:19:49,219 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:19:49,219 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,220 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:19:49,220 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(746): ClusterId : 9d0572e5-d9a1-4224-aef9-01c3d6dfc0be 2024-12-04T20:19:49,221 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:19:49,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:19:49,222 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,222 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:19:49,222 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:19:49,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:19:49,224 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:19:49,224 DEBUG [RS:0;c2ef38372881:37995 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20553bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:19:49,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:19:49,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:19:49,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:19:49,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:19:49,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740 2024-12-04T20:19:49,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740 2024-12-04T20:19:49,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:19:49,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:19:49,232 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:19:49,234 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:19:49,237 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:19:49,238 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=757853, jitterRate=-0.03634113073348999}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:19:49,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733343589216Initializing all the Stores at 1733343589217 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343589217Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343589217Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343589217Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343589217Cleaning up temporary data from old regions at 1733343589232 (+15 ms)Region opened successfully at 1733343589239 (+7 ms) 2024-12-04T20:19:49,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:19:49,239 DEBUG [RS:0;c2ef38372881:37995 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:37995 2024-12-04T20:19:49,239 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:19:49,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:19:49,240 INFO [RS:0;c2ef38372881:37995 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:19:49,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:19:49,240 INFO [RS:0;c2ef38372881:37995 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:19:49,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:19:49,240 DEBUG [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:19:49,240 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:19:49,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343589239Disabling compacts and flushes for region at 1733343589239Disabling writes for close at 1733343589240 (+1 ms)Writing region close event to WAL at 1733343589240Closed at 1733343589240 2024-12-04T20:19:49,241 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,44137,1733343588553 with port=37995, startcode=1733343588596 2024-12-04T20:19:49,241 DEBUG [RS:0;c2ef38372881:37995 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:19:49,242 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:19:49,242 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:19:49,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:19:49,244 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52973, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:19:49,244 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44137 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,245 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44137 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,245 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:19:49,246 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:19:49,247 DEBUG [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367 2024-12-04T20:19:49,247 DEBUG [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45475 2024-12-04T20:19:49,247 DEBUG [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:19:49,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:19:49,249 DEBUG [RS:0;c2ef38372881:37995 {}] zookeeper.ZKUtil(111): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,250 WARN [RS:0;c2ef38372881:37995 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:19:49,250 INFO [RS:0;c2ef38372881:37995 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:19:49,250 DEBUG [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/WALs/c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,250 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,37995,1733343588596] 2024-12-04T20:19:49,254 INFO [RS:0;c2ef38372881:37995 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:19:49,257 INFO [RS:0;c2ef38372881:37995 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:19:49,257 INFO [RS:0;c2ef38372881:37995 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:19:49,258 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,260 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:19:49,261 INFO [RS:0;c2ef38372881:37995 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:19:49,261 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,261 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,261 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:19:49,262 DEBUG [RS:0;c2ef38372881:37995 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:19:49,265 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,265 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,265 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,265 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,265 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,265 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37995,1733343588596-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:19:49,280 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:19:49,280 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37995,1733343588596-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,281 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,281 INFO [RS:0;c2ef38372881:37995 {}] regionserver.Replication(171): c2ef38372881,37995,1733343588596 started 2024-12-04T20:19:49,295 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,295 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,37995,1733343588596, RpcServer on c2ef38372881/172.17.0.2:37995, sessionid=0x100a6e38d280001 2024-12-04T20:19:49,295 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:19:49,295 DEBUG [RS:0;c2ef38372881:37995 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,295 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,37995,1733343588596' 2024-12-04T20:19:49,295 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:19:49,296 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:19:49,297 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:19:49,297 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:19:49,297 DEBUG [RS:0;c2ef38372881:37995 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,297 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,37995,1733343588596' 2024-12-04T20:19:49,297 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:19:49,297 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:19:49,298 DEBUG [RS:0;c2ef38372881:37995 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:19:49,298 INFO [RS:0;c2ef38372881:37995 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:19:49,298 INFO [RS:0;c2ef38372881:37995 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:19:49,397 WARN [c2ef38372881:44137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T20:19:49,401 INFO [RS:0;c2ef38372881:37995 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C37995%2C1733343588596, suffix=, logDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/WALs/c2ef38372881,37995,1733343588596, archiveDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/oldWALs, maxLogs=32 2024-12-04T20:19:49,405 INFO [RS:0;c2ef38372881:37995 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37995%2C1733343588596.1733343589404 2024-12-04T20:19:49,415 INFO [RS:0;c2ef38372881:37995 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/WALs/c2ef38372881,37995,1733343588596/c2ef38372881%2C37995%2C1733343588596.1733343589404 2024-12-04T20:19:49,417 DEBUG [RS:0;c2ef38372881:37995 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36427:36427),(127.0.0.1/127.0.0.1:41701:41701)] 2024-12-04T20:19:49,647 DEBUG [c2ef38372881:44137 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T20:19:49,648 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,650 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,37995,1733343588596, state=OPENING 2024-12-04T20:19:49,651 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:19:49,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,654 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:19:49,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:19:49,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:19:49,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,37995,1733343588596}] 2024-12-04T20:19:49,810 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:19:49,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50309, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:19:49,823 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:19:49,823 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:19:49,826 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C37995%2C1733343588596.meta, suffix=.meta, logDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/WALs/c2ef38372881,37995,1733343588596, archiveDir=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/oldWALs, maxLogs=32 2024-12-04T20:19:49,828 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37995%2C1733343588596.meta.1733343589828.meta 2024-12-04T20:19:49,835 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/WALs/c2ef38372881,37995,1733343588596/c2ef38372881%2C37995%2C1733343588596.meta.1733343589828.meta 2024-12-04T20:19:49,837 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36427:36427),(127.0.0.1/127.0.0.1:41701:41701)] 2024-12-04T20:19:49,838 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:19:49,839 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:19:49,839 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:19:49,839 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:19:49,839 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:19:49,839 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:49,839 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:19:49,839 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:19:49,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:19:49,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:19:49,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:19:49,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:19:49,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:19:49,846 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:19:49,846 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,847 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,847 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:19:49,848 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:19:49,848 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:49,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:49,848 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:19:49,849 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740 2024-12-04T20:19:49,851 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740 2024-12-04T20:19:49,852 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:19:49,852 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:19:49,853 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:19:49,855 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:19:49,855 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=716529, jitterRate=-0.0888867974281311}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:19:49,856 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:19:49,856 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733343589840Writing region info on filesystem at 1733343589840Initializing all the Stores at 1733343589841 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343589841Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343589841Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343589841Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343589841Cleaning up temporary data from old regions at 1733343589852 (+11 ms)Running coprocessor post-open hooks at 1733343589856 (+4 ms)Region opened successfully at 1733343589856 2024-12-04T20:19:49,858 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733343589809 2024-12-04T20:19:49,861 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:19:49,861 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:19:49,862 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,863 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,37995,1733343588596, state=OPEN 2024-12-04T20:19:49,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:19:49,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:19:49,865 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:19:49,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:19:49,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:19:49,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,37995,1733343588596 in 211 msec 2024-12-04T20:19:49,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:19:49,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 626 msec 2024-12-04T20:19:49,873 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:19:49,873 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:19:49,874 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:19:49,875 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,37995,1733343588596, seqNum=-1] 2024-12-04T20:19:49,875 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:19:49,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40785, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:19:49,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 694 msec 2024-12-04T20:19:49,884 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733343589884, completionTime=-1 2024-12-04T20:19:49,885 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T20:19:49,885 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:19:49,887 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T20:19:49,887 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733343649887 2024-12-04T20:19:49,887 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733343709887 2024-12-04T20:19:49,887 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-04T20:19:49,887 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,44137,1733343588553-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,887 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,44137,1733343588553-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,887 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,44137,1733343588553-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,887 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:44137, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,888 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,888 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,890 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:19:49,893 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.258sec 2024-12-04T20:19:49,893 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:19:49,893 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:19:49,893 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:19:49,893 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:19:49,893 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:19:49,893 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,44137,1733343588553-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:19:49,893 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,44137,1733343588553-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:19:49,896 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:19:49,896 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:19:49,896 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,44137,1733343588553-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:49,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205bfd42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:19:49,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,44137,-1 for getting cluster id 2024-12-04T20:19:49,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:19:49,923 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9d0572e5-d9a1-4224-aef9-01c3d6dfc0be' 2024-12-04T20:19:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:19:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9d0572e5-d9a1-4224-aef9-01c3d6dfc0be" 2024-12-04T20:19:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6e53c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:19:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,44137,-1] 2024-12-04T20:19:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:19:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:49,927 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:19:49,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dafc4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:19:49,930 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:19:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,37995,1733343588596, seqNum=-1] 2024-12-04T20:19:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:19:49,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:19:49,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,44137,1733343588553 2024-12-04T20:19:49,938 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:49,941 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T20:19:49,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:19:49,941 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:19:49,942 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:19:49,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:49,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:49,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:19:49,942 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:19:49,942 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1035653102, stopped=false 2024-12-04T20:19:49,943 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,44137,1733343588553 2024-12-04T20:19:49,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:19:49,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:19:49,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:49,944 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:19:49,944 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:19:49,944 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:19:49,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:49,944 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:19:49,945 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,37995,1733343588596' ***** 2024-12-04T20:19:49,945 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:19:49,945 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:19:49,945 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:19:49,945 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:19:49,945 INFO [RS:0;c2ef38372881:37995 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:19:49,945 INFO [RS:0;c2ef38372881:37995 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:19:49,945 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,37995,1733343588596 2024-12-04T20:19:49,945 INFO [RS:0;c2ef38372881:37995 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:19:49,945 INFO [RS:0;c2ef38372881:37995 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:37995. 2024-12-04T20:19:49,945 DEBUG [RS:0;c2ef38372881:37995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:19:49,945 DEBUG [RS:0;c2ef38372881:37995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:49,946 INFO [RS:0;c2ef38372881:37995 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:19:49,946 INFO [RS:0;c2ef38372881:37995 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:19:49,946 INFO [RS:0;c2ef38372881:37995 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:19:49,946 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:19:49,946 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T20:19:49,946 DEBUG [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T20:19:49,946 DEBUG [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:19:49,946 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:19:49,946 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:19:49,946 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:19:49,946 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:19:49,946 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:19:49,947 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-12-04T20:19:49,963 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740/.tmp/ns/4bc22e7568d040be927389c088aa6344 is 43, key is default/ns:d/1733343589877/Put/seqid=0 2024-12-04T20:19:49,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741835_1011 (size=5153) 2024-12-04T20:19:49,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741835_1011 (size=5153) 2024-12-04T20:19:49,970 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740/.tmp/ns/4bc22e7568d040be927389c088aa6344 2024-12-04T20:19:49,979 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740/.tmp/ns/4bc22e7568d040be927389c088aa6344 as hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740/ns/4bc22e7568d040be927389c088aa6344 2024-12-04T20:19:49,987 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740/ns/4bc22e7568d040be927389c088aa6344, entries=2, sequenceid=6, filesize=5.0 K 2024-12-04T20:19:49,988 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=6, compaction requested=false 2024-12-04T20:19:49,988 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T20:19:49,994 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T20:19:49,995 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:19:49,995 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:19:49,995 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343589946Running coprocessor pre-close hooks at 1733343589946Disabling compacts and flushes for region at 1733343589946Disabling writes for close at 1733343589946Obtaining lock to block concurrent updates at 1733343589947 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733343589947Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733343589947Flushing stores of hbase:meta,,1.1588230740 at 1733343589948 (+1 ms)Flushing 1588230740/ns: creating writer at 1733343589948Flushing 1588230740/ns: appending metadata at 1733343589963 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733343589963Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6be6c32d: reopening flushed file at 1733343589978 (+15 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=6, compaction requested=false at 1733343589988 (+10 ms)Writing region close event to WAL at 1733343589990 (+2 ms)Running coprocessor post-close hooks at 1733343589995 (+5 ms)Closed at 1733343589995 2024-12-04T20:19:49,995 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:19:50,146 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,37995,1733343588596; all regions closed. 2024-12-04T20:19:50,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741834_1010 (size=1152) 2024-12-04T20:19:50,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741834_1010 (size=1152) 2024-12-04T20:19:50,155 DEBUG [RS:0;c2ef38372881:37995 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/oldWALs 2024-12-04T20:19:50,156 INFO [RS:0;c2ef38372881:37995 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C37995%2C1733343588596.meta:.meta(num 1733343589828) 2024-12-04T20:19:50,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741833_1009 (size=93) 2024-12-04T20:19:50,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741833_1009 (size=93) 2024-12-04T20:19:50,163 DEBUG [RS:0;c2ef38372881:37995 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/oldWALs 2024-12-04T20:19:50,163 INFO [RS:0;c2ef38372881:37995 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C37995%2C1733343588596:(num 1733343589404) 2024-12-04T20:19:50,163 DEBUG [RS:0;c2ef38372881:37995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:50,163 INFO [RS:0;c2ef38372881:37995 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:19:50,163 INFO [RS:0;c2ef38372881:37995 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:19:50,163 INFO [RS:0;c2ef38372881:37995 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T20:19:50,163 INFO [RS:0;c2ef38372881:37995 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:19:50,163 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:19:50,164 INFO [RS:0;c2ef38372881:37995 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37995 2024-12-04T20:19:50,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:19:50,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,37995,1733343588596 2024-12-04T20:19:50,165 INFO [RS:0;c2ef38372881:37995 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:19:50,165 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007f9078903590@5ce4ceda rejected from java.util.concurrent.ThreadPoolExecutor@19e65b22[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-04T20:19:50,166 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,37995,1733343588596] 2024-12-04T20:19:50,167 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,37995,1733343588596 already deleted, retry=false 2024-12-04T20:19:50,167 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,37995,1733343588596 expired; onlineServers=0 2024-12-04T20:19:50,167 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,44137,1733343588553' ***** 2024-12-04T20:19:50,167 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:19:50,167 INFO [M:0;c2ef38372881:44137 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:19:50,167 INFO [M:0;c2ef38372881:44137 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:19:50,167 DEBUG [M:0;c2ef38372881:44137 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:19:50,167 DEBUG [M:0;c2ef38372881:44137 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:19:50,167 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:19:50,167 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343589198 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343589198,5,FailOnTimeoutGroup] 2024-12-04T20:19:50,167 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343589198 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343589198,5,FailOnTimeoutGroup] 2024-12-04T20:19:50,167 INFO [M:0;c2ef38372881:44137 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:19:50,167 INFO [M:0;c2ef38372881:44137 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:19:50,168 DEBUG [M:0;c2ef38372881:44137 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:19:50,168 INFO [M:0;c2ef38372881:44137 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:19:50,168 INFO [M:0;c2ef38372881:44137 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:19:50,168 INFO [M:0;c2ef38372881:44137 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:19:50,168 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:19:50,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:19:50,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:50,168 DEBUG [M:0;c2ef38372881:44137 {}] zookeeper.ZKUtil(347): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:19:50,168 WARN [M:0;c2ef38372881:44137 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:19:50,169 INFO [M:0;c2ef38372881:44137 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/.lastflushedseqids 2024-12-04T20:19:50,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741836_1012 (size=99) 2024-12-04T20:19:50,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741836_1012 (size=99) 2024-12-04T20:19:50,175 INFO [M:0;c2ef38372881:44137 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:19:50,175 INFO [M:0;c2ef38372881:44137 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:19:50,175 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:19:50,175 INFO [M:0;c2ef38372881:44137 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:50,175 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:50,175 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:19:50,175 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:50,176 INFO [M:0;c2ef38372881:44137 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-12-04T20:19:50,194 DEBUG [M:0;c2ef38372881:44137 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efb95153b39b42239157f3ec9d3a6c4a is 82, key is hbase:meta,,1/info:regioninfo/1733343589862/Put/seqid=0 2024-12-04T20:19:50,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741837_1013 (size=5672) 2024-12-04T20:19:50,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741837_1013 (size=5672) 2024-12-04T20:19:50,200 INFO [M:0;c2ef38372881:44137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efb95153b39b42239157f3ec9d3a6c4a 2024-12-04T20:19:50,221 DEBUG [M:0;c2ef38372881:44137 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2e950e11ffd543f8810076d247cdeee0 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733343589883/Put/seqid=0 2024-12-04T20:19:50,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741838_1014 (size=5275) 2024-12-04T20:19:50,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741838_1014 (size=5275) 2024-12-04T20:19:50,227 INFO [M:0;c2ef38372881:44137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2e950e11ffd543f8810076d247cdeee0 2024-12-04T20:19:50,246 DEBUG [M:0;c2ef38372881:44137 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11fe2f4fefcc4358ad2a49011764b214 is 69, key is c2ef38372881,37995,1733343588596/rs:state/1733343589245/Put/seqid=0 2024-12-04T20:19:50,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741839_1015 (size=5156) 2024-12-04T20:19:50,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741839_1015 (size=5156) 2024-12-04T20:19:50,252 INFO [M:0;c2ef38372881:44137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11fe2f4fefcc4358ad2a49011764b214 2024-12-04T20:19:50,266 INFO [RS:0;c2ef38372881:37995 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:19:50,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:19:50,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37995-0x100a6e38d280001, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:19:50,266 INFO [RS:0;c2ef38372881:37995 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,37995,1733343588596; zookeeper connection closed. 2024-12-04T20:19:50,266 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@170fc8f4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@170fc8f4 2024-12-04T20:19:50,267 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T20:19:50,274 DEBUG [M:0;c2ef38372881:44137 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fbbfb2b8432e455e963a8636849d9f00 is 52, key is load_balancer_on/state:d/1733343589940/Put/seqid=0 2024-12-04T20:19:50,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741840_1016 (size=5056) 2024-12-04T20:19:50,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741840_1016 (size=5056) 2024-12-04T20:19:50,279 INFO [M:0;c2ef38372881:44137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fbbfb2b8432e455e963a8636849d9f00 2024-12-04T20:19:50,286 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efb95153b39b42239157f3ec9d3a6c4a as hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/efb95153b39b42239157f3ec9d3a6c4a 2024-12-04T20:19:50,292 INFO [M:0;c2ef38372881:44137 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/efb95153b39b42239157f3ec9d3a6c4a, entries=8, sequenceid=29, filesize=5.5 K 2024-12-04T20:19:50,294 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2e950e11ffd543f8810076d247cdeee0 as hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2e950e11ffd543f8810076d247cdeee0 2024-12-04T20:19:50,300 INFO [M:0;c2ef38372881:44137 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2e950e11ffd543f8810076d247cdeee0, entries=3, sequenceid=29, filesize=5.2 K 2024-12-04T20:19:50,301 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11fe2f4fefcc4358ad2a49011764b214 as hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/11fe2f4fefcc4358ad2a49011764b214 2024-12-04T20:19:50,307 INFO [M:0;c2ef38372881:44137 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/11fe2f4fefcc4358ad2a49011764b214, entries=1, sequenceid=29, filesize=5.0 K 2024-12-04T20:19:50,308 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fbbfb2b8432e455e963a8636849d9f00 as hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fbbfb2b8432e455e963a8636849d9f00 2024-12-04T20:19:50,315 INFO [M:0;c2ef38372881:44137 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45475/user/jenkins/test-data/192432bc-9602-37f0-059b-ac8221d43367/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fbbfb2b8432e455e963a8636849d9f00, entries=1, sequenceid=29, filesize=4.9 K 2024-12-04T20:19:50,316 INFO [M:0;c2ef38372881:44137 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=29, compaction requested=false 2024-12-04T20:19:50,318 INFO [M:0;c2ef38372881:44137 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:50,318 DEBUG [M:0;c2ef38372881:44137 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343590175Disabling compacts and flushes for region at 1733343590175Disabling writes for close at 1733343590175Obtaining lock to block concurrent updates at 1733343590176 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733343590176Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733343590176Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733343590177 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733343590177Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733343590193 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733343590193Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733343590205 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733343590221 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733343590221Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733343590232 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733343590246 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733343590246Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733343590258 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733343590273 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733343590273Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@690cac9: reopening flushed file at 1733343590285 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@696a6d57: reopening flushed file at 1733343590293 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69af044d: reopening flushed file at 1733343590300 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71f1296e: reopening flushed file at 1733343590307 (+7 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=29, compaction requested=false at 1733343590316 (+9 ms)Writing region close event to WAL at 1733343590318 (+2 ms)Closed at 1733343590318 2024-12-04T20:19:50,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:19:50,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44465 is added to blk_1073741830_1006 (size=10311) 2024-12-04T20:19:50,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41875 is added to blk_1073741830_1006 (size=10311) 2024-12-04T20:19:50,322 INFO [M:0;c2ef38372881:44137 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:19:50,322 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:19:50,322 INFO [M:0;c2ef38372881:44137 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44137 2024-12-04T20:19:50,322 INFO [M:0;c2ef38372881:44137 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:19:50,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:19:50,424 INFO [M:0;c2ef38372881:44137 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:19:50,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44137-0x100a6e38d280000, quorum=127.0.0.1:58516, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:19:50,429 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d183c93{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:19:50,430 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7154ca22{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:19:50,430 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:19:50,430 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b44e274{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:19:50,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c1be80f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.log.dir/,STOPPED} 2024-12-04T20:19:50,433 WARN [BP-971878257-172.17.0.2-1733343587976 heartbeating to localhost/127.0.0.1:45475 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:19:50,433 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:19:50,434 WARN [BP-971878257-172.17.0.2-1733343587976 heartbeating to localhost/127.0.0.1:45475 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-971878257-172.17.0.2-1733343587976 (Datanode Uuid b6e8ce5b-c0e8-455a-b76f-e338c14b1d7b) service to localhost/127.0.0.1:45475 2024-12-04T20:19:50,434 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:19:50,435 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/data/data3/current/BP-971878257-172.17.0.2-1733343587976 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:19:50,435 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/data/data4/current/BP-971878257-172.17.0.2-1733343587976 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:19:50,436 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:19:50,437 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@542ee468{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:19:50,438 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75c88313{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:19:50,438 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:19:50,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4edee9ab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:19:50,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@eab7acc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.log.dir/,STOPPED} 2024-12-04T20:19:50,439 WARN [BP-971878257-172.17.0.2-1733343587976 heartbeating to localhost/127.0.0.1:45475 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:19:50,439 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:19:50,439 WARN [BP-971878257-172.17.0.2-1733343587976 heartbeating to localhost/127.0.0.1:45475 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-971878257-172.17.0.2-1733343587976 (Datanode Uuid 35a876a8-11e7-4cc9-b67a-7d0c5fcdca52) service to localhost/127.0.0.1:45475 2024-12-04T20:19:50,439 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:19:50,440 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/data/data1/current/BP-971878257-172.17.0.2-1733343587976 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:19:50,440 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/cluster_3bca46cb-6f5f-3cdd-ca59-c78efd86e91d/data/data2/current/BP-971878257-172.17.0.2-1733343587976 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:19:50,440 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:19:50,445 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49a88a00{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:19:50,446 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e4256d4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:19:50,446 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:19:50,446 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3197ca45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:19:50,446 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f681677{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.log.dir/,STOPPED} 2024-12-04T20:19:50,451 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:19:50,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T20:19:50,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.log.dir so I do NOT create it in target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f659524-c798-beaf-6eb5-d42d3f67ffe4/hadoop.tmp.dir so I do NOT create it in target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39, deleteOnExit=true 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/test.cache.data in system properties and HBase conf 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:19:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:19:50,467 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:19:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:19:50,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:19:50,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:19:50,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:19:50,479 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:19:50,530 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:19:50,535 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:19:50,536 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:19:50,536 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:19:50,536 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:19:50,536 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:19:50,537 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2305029e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:19:50,537 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a488aac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:19:50,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@195100a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/java.io.tmpdir/jetty-localhost-45381-hadoop-hdfs-3_4_1-tests_jar-_-any-3946549803162218775/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:19:50,626 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e349773{HTTP/1.1, (http/1.1)}{localhost:45381} 2024-12-04T20:19:50,626 INFO [Time-limited test {}] server.Server(415): Started @102466ms 2024-12-04T20:19:50,637 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:19:50,686 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:19:50,689 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:19:50,691 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:19:50,691 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:19:50,691 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:19:50,691 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48743db4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:19:50,692 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@257cf4bb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:19:50,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@692b8c40{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/java.io.tmpdir/jetty-localhost-39695-hadoop-hdfs-3_4_1-tests_jar-_-any-1413689736991847427/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:19:50,781 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1369ef4b{HTTP/1.1, (http/1.1)}{localhost:39695} 2024-12-04T20:19:50,781 INFO [Time-limited test {}] server.Server(415): Started @102621ms 2024-12-04T20:19:50,783 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:19:50,813 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:19:50,816 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:19:50,816 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:19:50,816 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:19:50,816 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:19:50,817 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ec1a06e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:19:50,817 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69bbaec1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:19:50,843 WARN [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data1/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:19:50,844 WARN [Thread-658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data2/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:19:50,859 WARN [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:19:50,862 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb0919e846ec3f4f with lease ID 0xabba01b09ec5951c: Processing first storage report for DS-0413b787-b062-40fb-aefd-17b90257faf5 from datanode DatanodeRegistration(127.0.0.1:43355, datanodeUuid=020ce560-df3f-45f1-a2c7-61a1d3514f1d, infoPort=44035, infoSecurePort=0, ipcPort=44951, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:19:50,862 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb0919e846ec3f4f with lease ID 0xabba01b09ec5951c: from storage DS-0413b787-b062-40fb-aefd-17b90257faf5 node DatanodeRegistration(127.0.0.1:43355, datanodeUuid=020ce560-df3f-45f1-a2c7-61a1d3514f1d, infoPort=44035, infoSecurePort=0, ipcPort=44951, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:50,862 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb0919e846ec3f4f with lease ID 0xabba01b09ec5951c: Processing first storage report for DS-da9ad50f-3c04-44a9-9c18-4200b22a8a38 from datanode DatanodeRegistration(127.0.0.1:43355, datanodeUuid=020ce560-df3f-45f1-a2c7-61a1d3514f1d, infoPort=44035, infoSecurePort=0, ipcPort=44951, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:19:50,862 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb0919e846ec3f4f with lease ID 0xabba01b09ec5951c: from storage DS-da9ad50f-3c04-44a9-9c18-4200b22a8a38 node DatanodeRegistration(127.0.0.1:43355, datanodeUuid=020ce560-df3f-45f1-a2c7-61a1d3514f1d, infoPort=44035, infoSecurePort=0, ipcPort=44951, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:50,911 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@f7d5275{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/java.io.tmpdir/jetty-localhost-43667-hadoop-hdfs-3_4_1-tests_jar-_-any-8072157725424777720/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:19:50,912 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5f783084{HTTP/1.1, (http/1.1)}{localhost:43667} 2024-12-04T20:19:50,912 INFO [Time-limited test {}] server.Server(415): Started @102752ms 2024-12-04T20:19:50,913 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:19:50,972 WARN [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data4/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:19:50,972 WARN [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data3/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:19:50,987 WARN [Thread-672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:19:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeefa98c37960d0f2 with lease ID 0xabba01b09ec5951d: Processing first storage report for DS-db585bb6-6935-4414-8f5b-90ce22b682f8 from datanode DatanodeRegistration(127.0.0.1:41997, datanodeUuid=da9387b2-45f0-494c-bf5b-3a8de868daf8, infoPort=36801, infoSecurePort=0, ipcPort=44895, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:19:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeefa98c37960d0f2 with lease ID 0xabba01b09ec5951d: from storage DS-db585bb6-6935-4414-8f5b-90ce22b682f8 node DatanodeRegistration(127.0.0.1:41997, datanodeUuid=da9387b2-45f0-494c-bf5b-3a8de868daf8, infoPort=36801, infoSecurePort=0, ipcPort=44895, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeefa98c37960d0f2 with lease ID 0xabba01b09ec5951d: Processing first storage report for DS-b33f0150-0138-452c-8e9b-b120f3f00cc4 from datanode DatanodeRegistration(127.0.0.1:41997, datanodeUuid=da9387b2-45f0-494c-bf5b-3a8de868daf8, infoPort=36801, infoSecurePort=0, ipcPort=44895, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:19:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeefa98c37960d0f2 with lease ID 0xabba01b09ec5951d: from storage DS-b33f0150-0138-452c-8e9b-b120f3f00cc4 node DatanodeRegistration(127.0.0.1:41997, datanodeUuid=da9387b2-45f0-494c-bf5b-3a8de868daf8, infoPort=36801, infoSecurePort=0, ipcPort=44895, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:19:51,038 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4 2024-12-04T20:19:51,041 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/zookeeper_0, clientPort=63536, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:19:51,042 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63536 2024-12-04T20:19:51,043 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:51,045 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:51,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:19:51,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:19:51,058 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d with version=8 2024-12-04T20:19:51,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase-staging 2024-12-04T20:19:51,060 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:19:51,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:51,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:51,061 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:19:51,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:51,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:19:51,061 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:19:51,061 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:19:51,062 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35483 2024-12-04T20:19:51,063 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35483 connecting to ZooKeeper ensemble=127.0.0.1:63536 2024-12-04T20:19:51,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:354830x0, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:19:51,068 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35483-0x100a6e396ef0000 connected 2024-12-04T20:19:51,090 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:51,092 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:51,094 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:19:51,094 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d, hbase.cluster.distributed=false 2024-12-04T20:19:51,096 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:19:51,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35483 2024-12-04T20:19:51,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35483 2024-12-04T20:19:51,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35483 2024-12-04T20:19:51,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35483 2024-12-04T20:19:51,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35483 2024-12-04T20:19:51,112 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:19:51,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:51,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:51,113 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:19:51,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:51,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:19:51,113 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:19:51,113 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:19:51,114 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34471 2024-12-04T20:19:51,115 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34471 connecting to ZooKeeper ensemble=127.0.0.1:63536 2024-12-04T20:19:51,115 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:51,117 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:51,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344710x0, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:19:51,121 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:344710x0, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:19:51,121 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34471-0x100a6e396ef0001 connected 2024-12-04T20:19:51,121 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:19:51,125 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:19:51,126 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:19:51,127 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:19:51,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34471 2024-12-04T20:19:51,129 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34471 2024-12-04T20:19:51,129 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34471 2024-12-04T20:19:51,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34471 2024-12-04T20:19:51,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34471 2024-12-04T20:19:51,145 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:35483 2024-12-04T20:19:51,145 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,35483,1733343591060 2024-12-04T20:19:51,266 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:19:51,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:19:51,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:19:51,343 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,35483,1733343591060 2024-12-04T20:19:51,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:19:51,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,556 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:19:51,556 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,35483,1733343591060 from backup master directory 2024-12-04T20:19:51,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,35483,1733343591060 2024-12-04T20:19:51,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:19:51,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:19:51,558 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:19:51,558 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,35483,1733343591060 2024-12-04T20:19:51,566 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/hbase.id] with ID: 463bd3fd-4285-476e-880c-360bbe302eeb 2024-12-04T20:19:51,566 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/.tmp/hbase.id 2024-12-04T20:19:51,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:19:51,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:19:51,574 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/.tmp/hbase.id]:[hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/hbase.id] 2024-12-04T20:19:51,589 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:51,589 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:19:51,590 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T20:19:51,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:19:51,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:19:51,600 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:19:51,601 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:19:51,601 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:19:51,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:19:51,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:19:51,610 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store 2024-12-04T20:19:51,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:19:51,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:19:51,618 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:51,619 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:19:51,619 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:51,619 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:51,619 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:19:51,619 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:51,619 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:19:51,619 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343591618Disabling compacts and flushes for region at 1733343591618Disabling writes for close at 1733343591619 (+1 ms)Writing region close event to WAL at 1733343591619Closed at 1733343591619 2024-12-04T20:19:51,620 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/.initializing 2024-12-04T20:19:51,620 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060 2024-12-04T20:19:51,622 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C35483%2C1733343591060, suffix=, logDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060, archiveDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/oldWALs, maxLogs=10 2024-12-04T20:19:51,623 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C35483%2C1733343591060.1733343591623 2024-12-04T20:19:51,628 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 2024-12-04T20:19:51,633 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36801:36801),(127.0.0.1/127.0.0.1:44035:44035)] 2024-12-04T20:19:51,634 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:19:51,634 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:51,634 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,634 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:19:51,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:51,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:19:51,640 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:19:51,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:19:51,643 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:19:51,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:19:51,646 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:19:51,646 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,647 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,648 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,649 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,649 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,650 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:19:51,651 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:19:51,654 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:19:51,655 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=822656, jitterRate=0.04606223106384277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:19:51,655 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733343591634Initializing all the Stores at 1733343591635 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343591635Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343591635Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343591635Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343591635Cleaning up temporary data from old regions at 1733343591649 (+14 ms)Region opened successfully at 1733343591655 (+6 ms) 2024-12-04T20:19:51,656 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:19:51,659 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@789c9e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:19:51,660 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:19:51,660 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:19:51,660 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:19:51,661 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:19:51,661 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T20:19:51,662 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T20:19:51,662 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:19:51,664 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:19:51,665 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:19:51,666 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:19:51,666 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:19:51,667 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:19:51,668 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:19:51,668 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:19:51,669 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:19:51,670 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:19:51,670 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:19:51,671 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:19:51,673 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:19:51,674 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:19:51,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:19:51,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:19:51,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,676 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,35483,1733343591060, sessionid=0x100a6e396ef0000, setting cluster-up flag (Was=false) 2024-12-04T20:19:51,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,680 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:19:51,681 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,35483,1733343591060 2024-12-04T20:19:51,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:51,687 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:19:51,688 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,35483,1733343591060 2024-12-04T20:19:51,689 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:19:51,691 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:19:51,691 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:19:51,691 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:19:51,691 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,35483,1733343591060 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:19:51,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:19:51,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:19:51,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:19:51,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:19:51,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:19:51,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:19:51,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,694 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733343621694 2024-12-04T20:19:51,694 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:19:51,694 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:19:51,694 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:19:51,694 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:19:51,694 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:19:51,694 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:19:51,695 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,695 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:19:51,695 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:19:51,695 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:19:51,695 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:19:51,696 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:19:51,696 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:19:51,696 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:19:51,696 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343591696,5,FailOnTimeoutGroup] 2024-12-04T20:19:51,697 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,697 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343591696,5,FailOnTimeoutGroup] 2024-12-04T20:19:51,697 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,697 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:19:51,697 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,697 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,697 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:19:51,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:19:51,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:19:51,710 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:19:51,710 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d 2024-12-04T20:19:51,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:19:51,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:19:51,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:51,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:19:51,726 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:19:51,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:51,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:19:51,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:19:51,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:51,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:19:51,730 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:19:51,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:51,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:19:51,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:19:51,733 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:51,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:51,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:19:51,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740 2024-12-04T20:19:51,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740 2024-12-04T20:19:51,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:19:51,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:19:51,737 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(746): ClusterId : 463bd3fd-4285-476e-880c-360bbe302eeb 2024-12-04T20:19:51,737 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:19:51,737 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:19:51,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:19:51,761 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:19:51,761 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:19:51,763 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:19:51,764 DEBUG [RS:0;c2ef38372881:34471 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19389fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:19:51,764 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:19:51,765 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692898, jitterRate=-0.11893559992313385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:19:51,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733343591723Initializing all the Stores at 1733343591724 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343591724Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343591724Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343591724Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343591724Cleaning up temporary data from old regions at 1733343591737 (+13 ms)Region opened successfully at 1733343591765 (+28 ms) 2024-12-04T20:19:51,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:19:51,766 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:19:51,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:19:51,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:19:51,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:19:51,766 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:19:51,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343591766Disabling compacts and flushes for region at 1733343591766Disabling writes for close at 1733343591766Writing region close event to WAL at 1733343591766Closed at 1733343591766 2024-12-04T20:19:51,768 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:19:51,768 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:19:51,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:19:51,770 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:19:51,772 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:19:51,776 DEBUG [RS:0;c2ef38372881:34471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:34471 2024-12-04T20:19:51,777 INFO [RS:0;c2ef38372881:34471 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:19:51,777 INFO [RS:0;c2ef38372881:34471 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:19:51,777 DEBUG [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:19:51,778 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,35483,1733343591060 with port=34471, startcode=1733343591112 2024-12-04T20:19:51,778 DEBUG [RS:0;c2ef38372881:34471 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:19:51,780 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40309, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:19:51,780 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35483 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,34471,1733343591112 2024-12-04T20:19:51,781 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35483 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,34471,1733343591112 2024-12-04T20:19:51,783 DEBUG [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d 2024-12-04T20:19:51,783 DEBUG [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40063 2024-12-04T20:19:51,783 DEBUG [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:19:51,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:19:51,785 DEBUG [RS:0;c2ef38372881:34471 {}] zookeeper.ZKUtil(111): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,34471,1733343591112 2024-12-04T20:19:51,785 WARN [RS:0;c2ef38372881:34471 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:19:51,785 INFO [RS:0;c2ef38372881:34471 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:19:51,785 DEBUG [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112 2024-12-04T20:19:51,785 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,34471,1733343591112] 2024-12-04T20:19:51,792 INFO [RS:0;c2ef38372881:34471 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:19:51,794 INFO [RS:0;c2ef38372881:34471 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:19:51,794 INFO [RS:0;c2ef38372881:34471 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:19:51,794 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,796 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:19:51,798 INFO [RS:0;c2ef38372881:34471 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:19:51,798 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,798 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,798 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,798 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:19:51,799 DEBUG [RS:0;c2ef38372881:34471 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:19:51,801 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,801 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,801 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,801 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,801 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,801 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,34471,1733343591112-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:19:51,815 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:19:51,815 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,34471,1733343591112-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,815 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,816 INFO [RS:0;c2ef38372881:34471 {}] regionserver.Replication(171): c2ef38372881,34471,1733343591112 started 2024-12-04T20:19:51,829 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:51,829 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,34471,1733343591112, RpcServer on c2ef38372881/172.17.0.2:34471, sessionid=0x100a6e396ef0001 2024-12-04T20:19:51,830 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:19:51,830 DEBUG [RS:0;c2ef38372881:34471 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,34471,1733343591112 2024-12-04T20:19:51,830 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,34471,1733343591112' 2024-12-04T20:19:51,830 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:19:51,831 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:19:51,831 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:19:51,831 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:19:51,831 DEBUG [RS:0;c2ef38372881:34471 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,34471,1733343591112 2024-12-04T20:19:51,831 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,34471,1733343591112' 2024-12-04T20:19:51,831 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:19:51,832 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:19:51,832 DEBUG [RS:0;c2ef38372881:34471 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:19:51,833 INFO [RS:0;c2ef38372881:34471 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:19:51,833 INFO [RS:0;c2ef38372881:34471 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:19:51,922 WARN [c2ef38372881:35483 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T20:19:51,935 INFO [RS:0;c2ef38372881:34471 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C34471%2C1733343591112, suffix=, logDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112, archiveDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs, maxLogs=32 2024-12-04T20:19:51,937 INFO [RS:0;c2ef38372881:34471 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34471%2C1733343591112.1733343591936 2024-12-04T20:19:51,943 INFO [RS:0;c2ef38372881:34471 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 2024-12-04T20:19:51,948 DEBUG [RS:0;c2ef38372881:34471 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44035:44035),(127.0.0.1/127.0.0.1:36801:36801)] 2024-12-04T20:19:52,173 DEBUG [c2ef38372881:35483 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T20:19:52,174 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,34471,1733343591112 2024-12-04T20:19:52,178 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,34471,1733343591112, state=OPENING 2024-12-04T20:19:52,181 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:19:52,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:52,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:19:52,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:19:52,185 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:19:52,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:19:52,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,34471,1733343591112}] 2024-12-04T20:19:52,340 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:19:52,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33301, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:19:52,352 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:19:52,352 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:19:52,354 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C34471%2C1733343591112.meta, suffix=.meta, logDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112, archiveDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs, maxLogs=32 2024-12-04T20:19:52,355 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta 2024-12-04T20:19:52,361 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta 2024-12-04T20:19:52,366 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44035:44035),(127.0.0.1/127.0.0.1:36801:36801)] 2024-12-04T20:19:52,367 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:19:52,367 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:19:52,367 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:19:52,367 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:19:52,367 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:19:52,367 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:52,367 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:19:52,368 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:19:52,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:19:52,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:19:52,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:52,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:52,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:19:52,371 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:19:52,371 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:52,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:52,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:19:52,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:19:52,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:52,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:52,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:19:52,374 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:19:52,374 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:52,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:19:52,375 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:19:52,376 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740 2024-12-04T20:19:52,377 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740 2024-12-04T20:19:52,378 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:19:52,378 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:19:52,379 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:19:52,381 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:19:52,382 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825499, jitterRate=0.04967673122882843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:19:52,382 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:19:52,382 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733343592368Writing region info on filesystem at 1733343592368Initializing all the Stores at 1733343592369 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343592369Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343592369Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343592369Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343592369Cleaning up temporary data from old regions at 1733343592378 (+9 ms)Running coprocessor post-open hooks at 1733343592382 (+4 ms)Region opened successfully at 1733343592382 2024-12-04T20:19:52,383 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733343592340 2024-12-04T20:19:52,386 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:19:52,386 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:19:52,387 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,34471,1733343591112 2024-12-04T20:19:52,388 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,34471,1733343591112, state=OPEN 2024-12-04T20:19:52,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:19:52,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:19:52,390 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,34471,1733343591112 2024-12-04T20:19:52,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:19:52,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:19:52,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:19:52,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,34471,1733343591112 in 205 msec 2024-12-04T20:19:52,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:19:52,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 625 msec 2024-12-04T20:19:52,396 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:19:52,396 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:19:52,397 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:19:52,397 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,34471,1733343591112, seqNum=-1] 2024-12-04T20:19:52,397 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:19:52,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48637, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:19:52,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 714 msec 2024-12-04T20:19:52,405 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733343592405, completionTime=-1 2024-12-04T20:19:52,405 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T20:19:52,405 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:19:52,407 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T20:19:52,407 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733343652407 2024-12-04T20:19:52,407 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733343712407 2024-12-04T20:19:52,407 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-04T20:19:52,407 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,35483,1733343591060-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,407 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,35483,1733343591060-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,407 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,35483,1733343591060-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,407 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:35483, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,408 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,408 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,410 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:19:52,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.852sec 2024-12-04T20:19:52,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:19:52,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:19:52,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:19:52,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:19:52,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:19:52,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,35483,1733343591060-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:19:52,411 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,35483,1733343591060-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:19:52,414 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:19:52,414 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:19:52,414 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,35483,1733343591060-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,428 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:52,432 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:52,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@597807df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:19:52,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,35483,-1 for getting cluster id 2024-12-04T20:19:52,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:19:52,439 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '463bd3fd-4285-476e-880c-360bbe302eeb' 2024-12-04T20:19:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:19:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "463bd3fd-4285-476e-880c-360bbe302eeb" 2024-12-04T20:19:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1323cbff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:19:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,35483,-1] 2024-12-04T20:19:52,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:19:52,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:19:52,443 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:19:52,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efce601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:19:52,444 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:19:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,34471,1733343591112, seqNum=-1] 2024-12-04T20:19:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:19:52,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:19:52,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,35483,1733343591060 2024-12-04T20:19:52,449 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:52,451 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T20:19:52,465 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:19:52,465 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:52,465 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:52,465 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:19:52,465 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:19:52,465 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:19:52,465 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:19:52,466 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:19:52,466 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37047 2024-12-04T20:19:52,467 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37047 connecting to ZooKeeper ensemble=127.0.0.1:63536 2024-12-04T20:19:52,468 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:52,469 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:19:52,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370470x0, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:19:52,472 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37047-0x100a6e396ef0002 connected 2024-12-04T20:19:52,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-04T20:19:52,472 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-04T20:19:52,473 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:19:52,473 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:19:52,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:19:52,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:19:52,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37047 2024-12-04T20:19:52,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37047 2024-12-04T20:19:52,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37047 2024-12-04T20:19:52,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37047 2024-12-04T20:19:52,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37047 2024-12-04T20:19:52,477 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(746): ClusterId : 463bd3fd-4285-476e-880c-360bbe302eeb 2024-12-04T20:19:52,477 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:19:52,479 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:19:52,479 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:19:52,481 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:19:52,481 DEBUG [RS:1;c2ef38372881:37047 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@153517fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:19:52,492 DEBUG [RS:1;c2ef38372881:37047 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c2ef38372881:37047 2024-12-04T20:19:52,492 INFO [RS:1;c2ef38372881:37047 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:19:52,492 INFO [RS:1;c2ef38372881:37047 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:19:52,492 DEBUG [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:19:52,493 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,35483,1733343591060 with port=37047, startcode=1733343592465 2024-12-04T20:19:52,493 DEBUG [RS:1;c2ef38372881:37047 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:19:52,495 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:19:52,495 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35483 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,37047,1733343592465 2024-12-04T20:19:52,495 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35483 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,37047,1733343592465 2024-12-04T20:19:52,496 DEBUG [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d 2024-12-04T20:19:52,496 DEBUG [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40063 2024-12-04T20:19:52,497 DEBUG [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:19:52,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:19:52,498 DEBUG [RS:1;c2ef38372881:37047 {}] zookeeper.ZKUtil(111): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,37047,1733343592465 2024-12-04T20:19:52,498 WARN [RS:1;c2ef38372881:37047 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:19:52,498 INFO [RS:1;c2ef38372881:37047 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:19:52,498 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,37047,1733343592465] 2024-12-04T20:19:52,498 DEBUG [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465 2024-12-04T20:19:52,502 INFO [RS:1;c2ef38372881:37047 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:19:52,503 INFO [RS:1;c2ef38372881:37047 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:19:52,504 INFO [RS:1;c2ef38372881:37047 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:19:52,504 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,504 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:19:52,505 INFO [RS:1;c2ef38372881:37047 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:19:52,505 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,505 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,505 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,505 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,505 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,505 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,505 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:19:52,505 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,505 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,506 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,506 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,506 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,506 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:19:52,506 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:19:52,506 DEBUG [RS:1;c2ef38372881:37047 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:19:52,506 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,506 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,506 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,506 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,506 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,506 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37047,1733343592465-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:19:52,520 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:19:52,521 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37047,1733343592465-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,521 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,521 INFO [RS:1;c2ef38372881:37047 {}] regionserver.Replication(171): c2ef38372881,37047,1733343592465 started 2024-12-04T20:19:52,532 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:19:52,532 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,37047,1733343592465, RpcServer on c2ef38372881/172.17.0.2:37047, sessionid=0x100a6e396ef0002 2024-12-04T20:19:52,532 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:19:52,532 DEBUG [RS:1;c2ef38372881:37047 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,37047,1733343592465 2024-12-04T20:19:52,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;c2ef38372881:37047,5,FailOnTimeoutGroup] 2024-12-04T20:19:52,532 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,37047,1733343592465' 2024-12-04T20:19:52,532 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:19:52,532 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-12-04T20:19:52,533 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:19:52,533 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:19:52,533 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:19:52,533 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:19:52,533 DEBUG [RS:1;c2ef38372881:37047 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,37047,1733343592465 2024-12-04T20:19:52,533 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,37047,1733343592465' 2024-12-04T20:19:52,533 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:19:52,534 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:19:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,35483,1733343591060 2024-12-04T20:19:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f9c1512 2024-12-04T20:19:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:19:52,534 DEBUG [RS:1;c2ef38372881:37047 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:19:52,534 INFO [RS:1;c2ef38372881:37047 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:19:52,534 INFO [RS:1;c2ef38372881:37047 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:19:52,536 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:19:52,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35483 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T20:19:52,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T20:19:52,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:19:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35483 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T20:19:52,540 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:19:52,540 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:52,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-12-04T20:19:52,541 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:19:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:19:52,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741835_1011 (size=393) 2024-12-04T20:19:52,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741835_1011 (size=393) 2024-12-04T20:19:52,549 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb4f1c9e49247cbbdbc1a36e5ba2d81f, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d 2024-12-04T20:19:52,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41997 is added to blk_1073741836_1012 (size=76) 2024-12-04T20:19:52,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43355 is added to blk_1073741836_1012 (size=76) 2024-12-04T20:19:52,556 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:52,556 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing eb4f1c9e49247cbbdbc1a36e5ba2d81f, disabling compactions & flushes 2024-12-04T20:19:52,556 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:19:52,556 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:19:52,556 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. after waiting 0 ms 2024-12-04T20:19:52,556 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:19:52,556 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:19:52,556 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: Waiting for close lock at 1733343592556Disabling compacts and flushes for region at 1733343592556Disabling writes for close at 1733343592556Writing region close event to WAL at 1733343592556Closed at 1733343592556 2024-12-04T20:19:52,558 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:19:52,558 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733343592558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733343592558"}]},"ts":"1733343592558"} 2024-12-04T20:19:52,561 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:19:52,563 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:19:52,563 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343592563"}]},"ts":"1733343592563"} 2024-12-04T20:19:52,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-12-04T20:19:52,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=eb4f1c9e49247cbbdbc1a36e5ba2d81f, ASSIGN}] 2024-12-04T20:19:52,568 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=eb4f1c9e49247cbbdbc1a36e5ba2d81f, ASSIGN 2024-12-04T20:19:52,569 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=eb4f1c9e49247cbbdbc1a36e5ba2d81f, ASSIGN; state=OFFLINE, location=c2ef38372881,34471,1733343591112; forceNewPlan=false, retain=false 2024-12-04T20:19:52,639 INFO [RS:1;c2ef38372881:37047 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C37047%2C1733343592465, suffix=, logDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465, archiveDir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs, maxLogs=32 2024-12-04T20:19:52,642 INFO [RS:1;c2ef38372881:37047 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37047%2C1733343592465.1733343592641 2024-12-04T20:19:52,649 INFO [RS:1;c2ef38372881:37047 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 2024-12-04T20:19:52,650 DEBUG [RS:1;c2ef38372881:37047 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44035:44035),(127.0.0.1/127.0.0.1:36801:36801)] 2024-12-04T20:19:52,720 INFO [c2ef38372881:35483 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T20:19:52,721 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eb4f1c9e49247cbbdbc1a36e5ba2d81f, regionState=OPENING, regionLocation=c2ef38372881,34471,1733343591112 2024-12-04T20:19:52,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=eb4f1c9e49247cbbdbc1a36e5ba2d81f, ASSIGN because future has completed 2024-12-04T20:19:52,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb4f1c9e49247cbbdbc1a36e5ba2d81f, server=c2ef38372881,34471,1733343591112}] 2024-12-04T20:19:52,892 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:19:52,893 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eb4f1c9e49247cbbdbc1a36e5ba2d81f, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:19:52,894 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,894 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:19:52,894 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,894 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,898 INFO [StoreOpener-eb4f1c9e49247cbbdbc1a36e5ba2d81f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,901 INFO [StoreOpener-eb4f1c9e49247cbbdbc1a36e5ba2d81f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb4f1c9e49247cbbdbc1a36e5ba2d81f columnFamilyName info 2024-12-04T20:19:52,901 DEBUG [StoreOpener-eb4f1c9e49247cbbdbc1a36e5ba2d81f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:19:52,902 INFO [StoreOpener-eb4f1c9e49247cbbdbc1a36e5ba2d81f-1 {}] regionserver.HStore(327): Store=eb4f1c9e49247cbbdbc1a36e5ba2d81f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:19:52,902 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,903 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,903 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,904 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,904 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,905 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,908 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:19:52,908 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eb4f1c9e49247cbbdbc1a36e5ba2d81f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743967, jitterRate=-0.05399823188781738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:19:52,908 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:19:52,909 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: Running coprocessor pre-open hook at 1733343592895Writing region info on filesystem at 1733343592895Initializing all the Stores at 1733343592897 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343592897Cleaning up temporary data from old regions at 1733343592904 (+7 ms)Running coprocessor post-open hooks at 1733343592908 (+4 ms)Region opened successfully at 1733343592909 (+1 ms) 2024-12-04T20:19:52,910 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f., pid=6, masterSystemTime=1733343592885 2024-12-04T20:19:52,912 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:19:52,912 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:19:52,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eb4f1c9e49247cbbdbc1a36e5ba2d81f, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,34471,1733343591112 2024-12-04T20:19:52,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb4f1c9e49247cbbdbc1a36e5ba2d81f, server=c2ef38372881,34471,1733343591112 because future has completed 2024-12-04T20:19:52,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T20:19:52,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eb4f1c9e49247cbbdbc1a36e5ba2d81f, server=c2ef38372881,34471,1733343591112 in 191 msec 2024-12-04T20:19:52,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T20:19:52,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=eb4f1c9e49247cbbdbc1a36e5ba2d81f, ASSIGN in 356 msec 2024-12-04T20:19:52,926 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:19:52,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343592926"}]},"ts":"1733343592926"} 2024-12-04T20:19:52,929 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-12-04T20:19:52,930 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:19:52,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 393 msec 2024-12-04T20:19:52,944 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T20:19:52,948 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:52,963 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:52,965 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:52,965 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:52,971 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T20:19:52,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-12-04T20:19:52,972 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-04T20:19:57,873 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T20:19:57,879 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:57,894 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:57,895 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:57,896 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:19:57,901 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-12-04T20:20:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:20:02,573 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-12-04T20:20:02,574 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-12-04T20:20:02,582 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T20:20:02,582 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:02,595 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:02,598 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:02,599 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:02,599 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:02,599 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:20:02,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ff5703b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:02,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36d0b5ff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:02,692 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9038e26{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/java.io.tmpdir/jetty-localhost-37373-hadoop-hdfs-3_4_1-tests_jar-_-any-640831341318428158/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:02,692 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@501a5826{HTTP/1.1, (http/1.1)}{localhost:37373} 2024-12-04T20:20:02,692 INFO [Time-limited test {}] server.Server(415): Started @114532ms 2024-12-04T20:20:02,694 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:02,724 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:02,727 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:02,728 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:02,728 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:02,728 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:20:02,728 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@167a7fde{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:02,728 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@712f5f14{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:02,757 WARN [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data5/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:02,757 WARN [Thread-830 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data6/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:02,777 WARN [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:02,779 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6ebe1be35340573 with lease ID 0xabba01b09ec5951e: Processing first storage report for DS-7809eccb-472e-44f0-8602-c9461cc12e54 from datanode DatanodeRegistration(127.0.0.1:42579, datanodeUuid=38736bf7-b2ee-4347-be9c-cefd6cc4f1b4, infoPort=43549, infoSecurePort=0, ipcPort=43303, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:20:02,779 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6ebe1be35340573 with lease ID 0xabba01b09ec5951e: from storage DS-7809eccb-472e-44f0-8602-c9461cc12e54 node DatanodeRegistration(127.0.0.1:42579, datanodeUuid=38736bf7-b2ee-4347-be9c-cefd6cc4f1b4, infoPort=43549, infoSecurePort=0, ipcPort=43303, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:02,780 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6ebe1be35340573 with lease ID 0xabba01b09ec5951e: Processing first storage report for DS-ef7f8ad4-0b80-4caa-aedc-f2750dad8b85 from datanode DatanodeRegistration(127.0.0.1:42579, datanodeUuid=38736bf7-b2ee-4347-be9c-cefd6cc4f1b4, infoPort=43549, infoSecurePort=0, ipcPort=43303, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:20:02,780 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6ebe1be35340573 with lease ID 0xabba01b09ec5951e: from storage DS-ef7f8ad4-0b80-4caa-aedc-f2750dad8b85 node DatanodeRegistration(127.0.0.1:42579, datanodeUuid=38736bf7-b2ee-4347-be9c-cefd6cc4f1b4, infoPort=43549, infoSecurePort=0, ipcPort=43303, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:02,827 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@272698f5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/java.io.tmpdir/jetty-localhost-35763-hadoop-hdfs-3_4_1-tests_jar-_-any-13826546640653763477/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:02,827 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15bbd738{HTTP/1.1, (http/1.1)}{localhost:35763} 2024-12-04T20:20:02,827 INFO [Time-limited test {}] server.Server(415): Started @114667ms 2024-12-04T20:20:02,828 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:02,861 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:02,865 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:02,866 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:02,866 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:02,866 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:20:02,867 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dc59954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:02,867 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68004957{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:02,892 WARN [Thread-865 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data8/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:02,892 WARN [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data7/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:02,912 WARN [Thread-844 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:02,915 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1fb253f7161ebae0 with lease ID 0xabba01b09ec5951f: Processing first storage report for DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8 from datanode DatanodeRegistration(127.0.0.1:36485, datanodeUuid=93efb62d-ea8f-4e8d-a76d-cc18b79ba75d, infoPort=44971, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:20:02,915 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1fb253f7161ebae0 with lease ID 0xabba01b09ec5951f: from storage DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8 node DatanodeRegistration(127.0.0.1:36485, datanodeUuid=93efb62d-ea8f-4e8d-a76d-cc18b79ba75d, infoPort=44971, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:02,915 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1fb253f7161ebae0 with lease ID 0xabba01b09ec5951f: Processing first storage report for DS-f885f06e-1f82-4eec-ade3-6f16d39374c2 from datanode DatanodeRegistration(127.0.0.1:36485, datanodeUuid=93efb62d-ea8f-4e8d-a76d-cc18b79ba75d, infoPort=44971, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:20:02,915 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1fb253f7161ebae0 with lease ID 0xabba01b09ec5951f: from storage DS-f885f06e-1f82-4eec-ade3-6f16d39374c2 node DatanodeRegistration(127.0.0.1:36485, datanodeUuid=93efb62d-ea8f-4e8d-a76d-cc18b79ba75d, infoPort=44971, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:02,966 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28246fba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/java.io.tmpdir/jetty-localhost-33521-hadoop-hdfs-3_4_1-tests_jar-_-any-3449868706172083380/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:02,967 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5056747b{HTTP/1.1, (http/1.1)}{localhost:33521} 2024-12-04T20:20:02,967 INFO [Time-limited test {}] server.Server(415): Started @114807ms 2024-12-04T20:20:02,968 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:03,023 WARN [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:03,023 WARN [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10/current/BP-1135270363-172.17.0.2-1733343590490/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:03,038 WARN [Thread-879 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8e877b1b55beafa2 with lease ID 0xabba01b09ec59520: Processing first storage report for DS-0ab71893-c5fb-41d0-ad81-a790574e6733 from datanode DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:20:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8e877b1b55beafa2 with lease ID 0xabba01b09ec59520: from storage DS-0ab71893-c5fb-41d0-ad81-a790574e6733 node DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8e877b1b55beafa2 with lease ID 0xabba01b09ec59520: Processing first storage report for DS-c978a0bf-c07c-41bb-8ab8-a7d0b89ddbcd from datanode DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490) 2024-12-04T20:20:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8e877b1b55beafa2 with lease ID 0xabba01b09ec59520: from storage DS-c978a0bf-c07c-41bb-8ab8-a7d0b89ddbcd node DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:03,085 WARN [ResponseProcessor for block BP-1135270363-172.17.0.2-1733343590490:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1135270363-172.17.0.2-1733343590490:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,086 WARN [ResponseProcessor for block BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,086 WARN [ResponseProcessor for block BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,086 WARN [ResponseProcessor for block BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013 java.io.IOException: Bad response ERROR for BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,087 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 block BP-1135270363-172.17.0.2-1733343590490:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:03,086 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 block BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK], DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:03,087 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 block BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK], DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:03,087 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta block BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK], DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:03,087 WARN [PacketResponder: BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41997] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,087 WARN [PacketResponder: BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41997] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:41464 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43355:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41464 dst: /127.0.0.1:43355 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:41458 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43355:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41458 dst: /127.0.0.1:43355 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1569855457_22 at /127.0.0.1:60280 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41997:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60280 dst: /127.0.0.1:41997 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_928081001_22 at /127.0.0.1:60330 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:41997:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60330 dst: /127.0.0.1:41997 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_928081001_22 at /127.0.0.1:41494 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:43355:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41494 dst: /127.0.0.1:43355 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,091 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1569855457_22 at /127.0.0.1:41452 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43355:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41452 dst: /127.0.0.1:43355 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,091 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:60306 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41997:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60306 dst: /127.0.0.1:41997 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,091 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:60302 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41997:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60302 dst: /127.0.0.1:41997 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,092 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@f7d5275{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:03,092 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5f783084{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:03,093 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:03,093 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69bbaec1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:03,093 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ec1a06e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:03,094 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:03,094 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:03,094 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1135270363-172.17.0.2-1733343590490 (Datanode Uuid da9387b2-45f0-494c-bf5b-3a8de868daf8) service to localhost/127.0.0.1:40063 2024-12-04T20:20:03,094 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:03,095 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data3/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:03,095 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data4/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:03,095 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:03,098 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@f98807 {}] datanode.DataXceiver(331): 127.0.0.1:43355:DataXceiver error processing unknown operation src: /127.0.0.1:33248 dst: /127.0.0.1:43355 java.io.IOException: Server closed. at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:03,098 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta block BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,098 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 block BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,099 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 block BP-1135270363-172.17.0.2-1733343590490:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,099 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 block BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,100 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@692b8c40{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:03,101 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1369ef4b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:03,101 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:03,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@257cf4bb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:03,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48743db4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:03,102 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:03,102 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:03,102 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1135270363-172.17.0.2-1733343590490 (Datanode Uuid 020ce560-df3f-45f1-a2c7-61a1d3514f1d) service to localhost/127.0.0.1:40063 2024-12-04T20:20:03,102 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:03,103 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data1/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:03,103 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data2/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:03,103 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f., hostname=c2ef38372881,34471,1733343591112, seqNum=2] 2024-12-04T20:20:03,107 ERROR [FSHLog-0-hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d-prefix:c2ef38372881,34471,1733343591112 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,107 WARN [FSHLog-0-hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d-prefix:c2ef38372881,34471,1733343591112 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,108 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,108 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C34471%2C1733343591112:(num 1733343591936) roll requested 2024-12-04T20:20:03,108 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34471%2C1733343591112.1733343603108 2024-12-04T20:20:03,110 WARN [Thread-902 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,111 WARN [Thread-902 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:03,111 WARN [Thread-902 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741838_1018 2024-12-04T20:20:03,113 WARN [Thread-902 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:03,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:03,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:03,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:03,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:03,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:03,119 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343603108 2024-12-04T20:20:03,119 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,120 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:03,121 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-04T20:20:03,121 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-04T20:20:03,121 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43549:43549),(127.0.0.1/127.0.0.1:44971:44971)] 2024-12-04T20:20:03,121 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 2024-12-04T20:20:03,121 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 is not closed yet, will try archiving it next time 2024-12-04T20:20:03,124 WARN [IPC Server handler 1 on default port 40063 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1009 2024-12-04T20:20:03,128 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 after 5ms 2024-12-04T20:20:03,603 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:04,507 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:05,121 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:05,123 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343603108 2024-12-04T20:20:05,124 WARN [ResponseProcessor for block BP-1135270363-172.17.0.2-1733343590490:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1135270363-172.17.0.2-1733343590490:blk_1073741839_1019 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:05,125 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343603108 block BP-1135270363-172.17.0.2-1733343590490:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK], DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:05,126 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:56770 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:42579:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56770 dst: /127.0.0.1:42579 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:05,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:40606 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:36485:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40606 dst: /127.0.0.1:36485 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:05,129 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9038e26{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:05,129 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@501a5826{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:05,130 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:05,130 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36d0b5ff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:05,130 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ff5703b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:05,133 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:05,133 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:05,133 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1135270363-172.17.0.2-1733343590490 (Datanode Uuid 38736bf7-b2ee-4347-be9c-cefd6cc4f1b4) service to localhost/127.0.0.1:40063 2024-12-04T20:20:05,133 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:05,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data5/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:05,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data6/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:05,134 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:05,603 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:06,508 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:07,123 WARN [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]] 2024-12-04T20:20:07,123 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:07,124 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C34471%2C1733343591112:(num 1733343603108) roll requested 2024-12-04T20:20:07,124 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34471%2C1733343591112.1733343607124 2024-12-04T20:20:07,130 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 after 4009ms 2024-12-04T20:20:07,133 WARN [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42579 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:07,133 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:40632 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741840_1022] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data8]'}, localName='127.0.0.1:36485', datanodeUuid='93efb62d-ea8f-4e8d-a76d-cc18b79ba75d', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741840_1022 to mirror 127.0.0.1:42579 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:07,133 WARN [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:07,133 WARN [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741840_1022 2024-12-04T20:20:07,133 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:40632 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741840_1022] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T20:20:07,133 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:40632 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:36485:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40632 dst: /127.0.0.1:36485 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:07,134 WARN [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:07,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:07,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:07,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:07,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:07,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:07,139 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T20:20:07,139 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343603108 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343607124 2024-12-04T20:20:07,140 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44971:44971),(127.0.0.1/127.0.0.1:35487:35487)] 2024-12-04T20:20:07,140 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 is not closed yet, will try archiving it next time 2024-12-04T20:20:07,140 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343603108 is not closed yet, will try archiving it next time 2024-12-04T20:20:07,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36485 is added to blk_1073741839_1021 (size=2431) 2024-12-04T20:20:07,544 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 is not closed yet, will try archiving it next time 2024-12-04T20:20:07,604 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:08,508 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:08,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741839_1021 (size=2431) 2024-12-04T20:20:09,140 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,143 WARN [ResponseProcessor for block BP-1135270363-172.17.0.2-1733343590490:blk_1073741841_1023 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1135270363-172.17.0.2-1733343590490:blk_1073741841_1023 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,144 WARN [DataStreamer for file /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343607124 block BP-1135270363-172.17.0.2-1733343590490:blk_1073741841_1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:09,145 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:40634 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:36485:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40634 dst: /127.0.0.1:36485 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,146 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:56214 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56214 dst: /127.0.0.1:45187 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,149 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@272698f5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:09,150 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15bbd738{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:09,150 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:09,150 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@712f5f14{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:09,150 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@167a7fde{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:09,151 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:09,152 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:09,152 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:09,152 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1135270363-172.17.0.2-1733343590490 (Datanode Uuid 93efb62d-ea8f-4e8d-a76d-cc18b79ba75d) service to localhost/127.0.0.1:40063 2024-12-04T20:20:09,152 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data7/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:09,152 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data8/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:09,152 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] regionserver.HRegion(8855): Flush requested on eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:09,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb4f1c9e49247cbbdbc1a36e5ba2d81f 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:20:09,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/887f46696b2b4854a4659723968d957c is 1080, key is row0002/info:/1733343605136/Put/seqid=0 2024-12-04T20:20:09,183 WARN [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41997 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,183 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48290 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741842_1025] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741842_1025 to mirror 127.0.0.1:41997 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,183 WARN [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:09,183 WARN [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741842_1025 2024-12-04T20:20:09,183 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48290 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741842_1025] {}] datanode.BlockReceiver(316): Block 1073741842 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:09,183 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48290 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48290 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,184 WARN [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:09,185 WARN [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,185 WARN [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:09,185 WARN [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741843_1026 2024-12-04T20:20:09,186 WARN [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:09,187 WARN [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,187 WARN [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:09,187 WARN [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741844_1027 2024-12-04T20:20:09,188 WARN [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:09,190 WARN [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42579 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,190 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48302 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741845_1028] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741845_1028 to mirror 127.0.0.1:42579 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,190 WARN [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:09,190 WARN [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741845_1028 2024-12-04T20:20:09,190 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48302 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741845_1028] {}] datanode.BlockReceiver(316): Block 1073741845 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:09,190 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48302 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741845_1028] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48302 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,191 WARN [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:09,191 WARN [IPC Server handler 1 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:09,191 WARN [IPC Server handler 1 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:09,192 WARN [IPC Server handler 1 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:09,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741846_1029 (size=10347) 2024-12-04T20:20:09,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/887f46696b2b4854a4659723968d957c 2024-12-04T20:20:09,605 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/887f46696b2b4854a4659723968d957c as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/887f46696b2b4854a4659723968d957c 2024-12-04T20:20:09,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/887f46696b2b4854a4659723968d957c, entries=5, sequenceid=11, filesize=10.1 K 2024-12-04T20:20:09,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for eb4f1c9e49247cbbdbc1a36e5ba2d81f in 456ms, sequenceid=11, compaction requested=false 2024-12-04T20:20:09,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] regionserver.HRegion(8855): Flush requested on eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:09,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb4f1c9e49247cbbdbc1a36e5ba2d81f 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-12-04T20:20:09,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/a0054509a5a04ace9a02652f3867bc32 is 1080, key is row0007/info:/1733343609163/Put/seqid=0 2024-12-04T20:20:09,799 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48334 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741847_1030 to mirror 127.0.0.1:41997 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,800 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41997 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,800 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48334 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:09,800 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:09,800 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741847_1030 2024-12-04T20:20:09,800 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48334 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48334 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,801 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:09,803 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,803 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK], DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:09,803 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741848_1031 2024-12-04T20:20:09,804 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:09,805 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,806 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:09,806 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741849_1032 2024-12-04T20:20:09,806 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:09,809 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42579 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:09,809 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48346 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741850_1033] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741850_1033 to mirror 127.0.0.1:42579 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,809 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:09,809 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741850_1033 2024-12-04T20:20:09,809 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48346 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741850_1033] {}] datanode.BlockReceiver(316): Block 1073741850 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:09,809 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48346 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741850_1033] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48346 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:09,810 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:09,811 WARN [IPC Server handler 1 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:09,811 WARN [IPC Server handler 1 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:09,811 WARN [IPC Server handler 1 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:09,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741851_1034 (size=12506) 2024-12-04T20:20:10,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/a0054509a5a04ace9a02652f3867bc32 2024-12-04T20:20:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/a0054509a5a04ace9a02652f3867bc32 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/a0054509a5a04ace9a02652f3867bc32 2024-12-04T20:20:10,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/a0054509a5a04ace9a02652f3867bc32, entries=7, sequenceid=24, filesize=12.2 K 2024-12-04T20:20:10,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for eb4f1c9e49247cbbdbc1a36e5ba2d81f in 438ms, sequenceid=24, compaction requested=false 2024-12-04T20:20:10,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:10,231 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-12-04T20:20:10,231 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:10,231 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/a0054509a5a04ace9a02652f3867bc32 because midkey is the same as first or last row 2024-12-04T20:20:10,509 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,141 WARN [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]] 2024-12-04T20:20:11,141 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,142 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C34471%2C1733343591112:(num 1733343607124) roll requested 2024-12-04T20:20:11,143 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34471%2C1733343591112.1733343611142 2024-12-04T20:20:11,150 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41997 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,150 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48358 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741852_1035] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741852_1035 to mirror 127.0.0.1:41997 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,151 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:11,151 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48358 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741852_1035] {}] datanode.BlockReceiver(316): Block 1073741852 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T20:20:11,151 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741852_1035 2024-12-04T20:20:11,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48358 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741852_1035] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48358 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,152 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:11,154 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,154 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:11,154 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741853_1036 2024-12-04T20:20:11,155 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:11,156 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,156 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:11,156 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741854_1037 2024-12-04T20:20:11,157 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:11,158 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,158 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:11,158 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741855_1038 2024-12-04T20:20:11,159 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:11,160 WARN [IPC Server handler 3 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:11,160 WARN [IPC Server handler 3 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:11,160 WARN [IPC Server handler 3 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:11,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:11,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:11,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:11,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:11,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:11,163 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343607124 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343611142 2024-12-04T20:20:11,164 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35487:35487)] 2024-12-04T20:20:11,164 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 is not closed yet, will try archiving it next time 2024-12-04T20:20:11,164 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343607124 is not closed yet, will try archiving it next time 2024-12-04T20:20:11,165 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343603108 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs/c2ef38372881%2C34471%2C1733343591112.1733343603108 2024-12-04T20:20:11,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741841_1024 (size=25992) 2024-12-04T20:20:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] regionserver.HRegion(8855): Flush requested on eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:11,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb4f1c9e49247cbbdbc1a36e5ba2d81f 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-04T20:20:11,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/446cbf45e98947d38b854508a39240d3 is 1079, key is tmprow/info:/1733343611223/Put/seqid=0 2024-12-04T20:20:11,235 WARN [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,236 WARN [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:11,236 WARN [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741857_1040 2024-12-04T20:20:11,236 WARN [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:11,237 WARN [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,237 WARN [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:11,237 WARN [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741858_1041 2024-12-04T20:20:11,238 WARN [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:11,239 WARN [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,239 WARN [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:11,239 WARN [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741859_1042 2024-12-04T20:20:11,240 WARN [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:11,242 WARN [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43355 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,242 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48382 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741860_1043 to mirror 127.0.0.1:43355 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,242 WARN [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:11,242 WARN [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741860_1043 2024-12-04T20:20:11,242 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48382 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:11,242 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48382 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48382 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,243 WARN [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:11,243 WARN [IPC Server handler 4 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:11,243 WARN [IPC Server handler 4 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:11,243 WARN [IPC Server handler 4 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:11,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741861_1044 (size=6027) 2024-12-04T20:20:11,567 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 is not closed yet, will try archiving it next time 2024-12-04T20:20:11,605 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/446cbf45e98947d38b854508a39240d3 2024-12-04T20:20:11,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/446cbf45e98947d38b854508a39240d3 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/446cbf45e98947d38b854508a39240d3 2024-12-04T20:20:11,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/446cbf45e98947d38b854508a39240d3, entries=1, sequenceid=34, filesize=5.9 K 2024-12-04T20:20:11,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for eb4f1c9e49247cbbdbc1a36e5ba2d81f in 436ms, sequenceid=34, compaction requested=true 2024-12-04T20:20:11,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:11,661 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-12-04T20:20:11,661 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/a0054509a5a04ace9a02652f3867bc32 because midkey is the same as first or last row 2024-12-04T20:20:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb4f1c9e49247cbbdbc1a36e5ba2d81f:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:20:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:20:11,662 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:20:11,664 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:20:11,664 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1541): eb4f1c9e49247cbbdbc1a36e5ba2d81f/info is initiating minor compaction (all files) 2024-12-04T20:20:11,664 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb4f1c9e49247cbbdbc1a36e5ba2d81f/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:11,664 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/887f46696b2b4854a4659723968d957c, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/a0054509a5a04ace9a02652f3867bc32, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/446cbf45e98947d38b854508a39240d3] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp, totalSize=28.2 K 2024-12-04T20:20:11,664 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting 887f46696b2b4854a4659723968d957c, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733343605136 2024-12-04T20:20:11,665 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0054509a5a04ace9a02652f3867bc32, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733343609163 2024-12-04T20:20:11,665 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting 446cbf45e98947d38b854508a39240d3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733343611223 2024-12-04T20:20:11,678 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb4f1c9e49247cbbdbc1a36e5ba2d81f#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:20:11,678 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/b3d98c38ce0f42989684347d247e8c45 is 1080, key is row0002/info:/1733343605136/Put/seqid=0 2024-12-04T20:20:11,680 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,680 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:11,680 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741862_1045 2024-12-04T20:20:11,681 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:11,682 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41997 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,682 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48408 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741863_1046 to mirror 127.0.0.1:41997 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,683 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:11,683 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741863_1046 2024-12-04T20:20:11,683 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48408 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:11,683 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48408 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48408 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,683 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:11,685 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42579 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,686 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:11,685 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48410 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741864_1047 to mirror 127.0.0.1:42579 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,686 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741864_1047 2024-12-04T20:20:11,686 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48410 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:11,686 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48410 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48410 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,686 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:11,688 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43355 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:11,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48416 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741865_1048 to mirror 127.0.0.1:43355 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,688 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:11,688 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741865_1048 2024-12-04T20:20:11,688 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48416 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:11,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48416 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48416 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:11,689 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:11,689 WARN [IPC Server handler 1 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:11,689 WARN [IPC Server handler 1 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:11,690 WARN [IPC Server handler 1 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:11,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741866_1049 (size=17994) 2024-12-04T20:20:12,044 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@77f37dae[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741846_1029 to 127.0.0.1:43355 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:12,045 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@79349d20[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741851_1034 to 127.0.0.1:41997 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:12,105 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/b3d98c38ce0f42989684347d247e8c45 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45 2024-12-04T20:20:12,113 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb4f1c9e49247cbbdbc1a36e5ba2d81f/info of eb4f1c9e49247cbbdbc1a36e5ba2d81f into b3d98c38ce0f42989684347d247e8c45(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:20:12,113 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:12,113 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f., storeName=eb4f1c9e49247cbbdbc1a36e5ba2d81f/info, priority=13, startTime=1733343611662; duration=0sec 2024-12-04T20:20:12,113 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-04T20:20:12,113 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45 because midkey is the same as first or last row 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45 because midkey is the same as first or last row 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45 because midkey is the same as first or last row 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:20:12,114 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb4f1c9e49247cbbdbc1a36e5ba2d81f:info 2024-12-04T20:20:12,510 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] regionserver.HRegion(8855): Flush requested on eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:12,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb4f1c9e49247cbbdbc1a36e5ba2d81f 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-04T20:20:12,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/3dac2af668c149b38de215c63181733d is 1079, key is tmprow/info:/1733343612655/Put/seqid=0 2024-12-04T20:20:12,666 WARN [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:12,667 WARN [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:12,667 WARN [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741867_1050 2024-12-04T20:20:12,667 WARN [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:12,668 WARN [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:12,668 WARN [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:12,668 WARN [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741868_1051 2024-12-04T20:20:12,669 WARN [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:12,670 WARN [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:12,670 WARN [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:12,670 WARN [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741869_1052 2024-12-04T20:20:12,671 WARN [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:12,672 WARN [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:12,672 WARN [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:12,672 WARN [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741870_1053 2024-12-04T20:20:12,673 WARN [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:12,674 WARN [IPC Server handler 0 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:12,674 WARN [IPC Server handler 0 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:12,674 WARN [IPC Server handler 0 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:12,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741871_1054 (size=6027) 2024-12-04T20:20:13,046 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@79349d20[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741841_1024 to 127.0.0.1:42579 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:13,046 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@77f37dae[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741861_1044 to 127.0.0.1:36485 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:13,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/3dac2af668c149b38de215c63181733d 2024-12-04T20:20:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/3dac2af668c149b38de215c63181733d as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/3dac2af668c149b38de215c63181733d 2024-12-04T20:20:13,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/3dac2af668c149b38de215c63181733d, entries=1, sequenceid=45, filesize=5.9 K 2024-12-04T20:20:13,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for eb4f1c9e49247cbbdbc1a36e5ba2d81f in 442ms, sequenceid=45, compaction requested=false 2024-12-04T20:20:13,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:13,101 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-12-04T20:20:13,101 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:13,101 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45 because midkey is the same as first or last row 2024-12-04T20:20:13,165 WARN [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]] 2024-12-04T20:20:13,165 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:13,165 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C34471%2C1733343591112:(num 1733343611142) roll requested 2024-12-04T20:20:13,166 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34471%2C1733343591112.1733343613165 2024-12-04T20:20:13,170 WARN [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:13,170 WARN [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:13,170 WARN [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741872_1055 2024-12-04T20:20:13,171 WARN [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:13,173 WARN [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:13,173 WARN [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:13,173 WARN [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741873_1056 2024-12-04T20:20:13,174 WARN [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:13,178 WARN [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43355 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:13,178 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48458 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741874_1057 to mirror 127.0.0.1:43355 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:13,178 WARN [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:13,178 WARN [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741874_1057 2024-12-04T20:20:13,178 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48458 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T20:20:13,179 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48458 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48458 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:13,179 WARN [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:13,183 WARN [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41997 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:13,183 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48462 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741875_1058 to mirror 127.0.0.1:41997 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:13,183 WARN [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:13,183 WARN [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741875_1058 2024-12-04T20:20:13,183 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48462 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T20:20:13,183 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48462 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48462 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:13,184 WARN [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:13,184 WARN [IPC Server handler 0 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:13,184 WARN [IPC Server handler 0 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:13,185 WARN [IPC Server handler 0 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:13,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:13,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:13,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:13,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:13,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:13,188 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343611142 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343613165 2024-12-04T20:20:13,189 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35487:35487)] 2024-12-04T20:20:13,189 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 is not closed yet, will try archiving it next time 2024-12-04T20:20:13,189 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343611142 is not closed yet, will try archiving it next time 2024-12-04T20:20:13,190 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343607124 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs/c2ef38372881%2C34471%2C1733343591112.1733343607124 2024-12-04T20:20:13,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741856_1039 (size=13591) 2024-12-04T20:20:13,593 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 is not closed yet, will try archiving it next time 2024-12-04T20:20:13,606 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] regionserver.HRegion(8855): Flush requested on eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:14,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb4f1c9e49247cbbdbc1a36e5ba2d81f 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-04T20:20:14,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2 is 1079, key is tmprow/info:/1733343614090/Put/seqid=0 2024-12-04T20:20:14,105 WARN [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36485 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48480 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741877_1060 to mirror 127.0.0.1:36485 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,106 WARN [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:14,106 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48480 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:14,106 WARN [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741877_1060 2024-12-04T20:20:14,106 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48480 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48480 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,106 WARN [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:14,108 WARN [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,108 WARN [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:14,108 WARN [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741878_1061 2024-12-04T20:20:14,108 WARN [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:14,109 WARN [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,110 WARN [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:14,110 WARN [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741879_1062 2024-12-04T20:20:14,110 WARN [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:14,112 WARN [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43355 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,112 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48488 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741880_1063] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741880_1063 to mirror 127.0.0.1:43355 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,112 WARN [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:14,112 WARN [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741880_1063 2024-12-04T20:20:14,112 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48488 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741880_1063] {}] datanode.BlockReceiver(316): Block 1073741880 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:14,112 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48488 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741880_1063] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48488 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,113 WARN [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:14,114 WARN [IPC Server handler 3 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:14,114 WARN [IPC Server handler 3 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:14,114 WARN [IPC Server handler 3 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:14,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741881_1064 (size=6027) 2024-12-04T20:20:14,511 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2 2024-12-04T20:20:14,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2 2024-12-04T20:20:14,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2, entries=1, sequenceid=55, filesize=5.9 K 2024-12-04T20:20:14,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for eb4f1c9e49247cbbdbc1a36e5ba2d81f in 448ms, sequenceid=55, compaction requested=true 2024-12-04T20:20:14,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:14,540 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K 2024-12-04T20:20:14,540 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:14,540 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45 because midkey is the same as first or last row 2024-12-04T20:20:14,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb4f1c9e49247cbbdbc1a36e5ba2d81f:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:20:14,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:20:14,540 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:20:14,542 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:20:14,542 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1541): eb4f1c9e49247cbbdbc1a36e5ba2d81f/info is initiating minor compaction (all files) 2024-12-04T20:20:14,542 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb4f1c9e49247cbbdbc1a36e5ba2d81f/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:14,542 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/3dac2af668c149b38de215c63181733d, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp, totalSize=29.3 K 2024-12-04T20:20:14,542 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3d98c38ce0f42989684347d247e8c45, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733343605136 2024-12-04T20:20:14,543 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dac2af668c149b38de215c63181733d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733343612655 2024-12-04T20:20:14,543 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7a0e4e35f24d3daaf79ce25e6c2ff2, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733343614090 2024-12-04T20:20:14,558 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb4f1c9e49247cbbdbc1a36e5ba2d81f#info#compaction#24 average throughput is 12.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:20:14,559 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/bd5d6f5727fc42c3a4b6418a8a05e588 is 1080, key is row0002/info:/1733343605136/Put/seqid=0 2024-12-04T20:20:14,561 WARN [Thread-976 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43355 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,561 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48512 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741882_1065] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741882_1065 to mirror 127.0.0.1:43355 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,561 WARN [Thread-976 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]) is bad. 2024-12-04T20:20:14,561 WARN [Thread-976 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741882_1065 2024-12-04T20:20:14,561 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48512 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741882_1065] {}] datanode.BlockReceiver(316): Block 1073741882 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:14,561 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48512 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741882_1065] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48512 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,562 WARN [Thread-976 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK] 2024-12-04T20:20:14,563 WARN [Thread-976 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,563 WARN [Thread-976 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]) is bad. 2024-12-04T20:20:14,563 WARN [Thread-976 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741883_1066 2024-12-04T20:20:14,563 WARN [Thread-976 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41997,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK] 2024-12-04T20:20:14,565 WARN [Thread-976 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36485 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,565 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48524 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741884_1067] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741884_1067 to mirror 127.0.0.1:36485 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,565 WARN [Thread-976 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:14,566 WARN [Thread-976 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741884_1067 2024-12-04T20:20:14,566 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48524 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741884_1067] {}] datanode.BlockReceiver(316): Block 1073741884 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:14,566 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48524 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741884_1067] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48524 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,566 WARN [Thread-976 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:14,568 WARN [Thread-976 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42579 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:14,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48532 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741885_1068] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741885_1068 to mirror 127.0.0.1:42579 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,568 WARN [Thread-976 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK]) is bad. 2024-12-04T20:20:14,568 WARN [Thread-976 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741885_1068 2024-12-04T20:20:14,568 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48532 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741885_1068] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:14,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:48532 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741885_1068] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48532 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:14,569 WARN [Thread-976 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42579,DS-7809eccb-472e-44f0-8602-c9461cc12e54,DISK] 2024-12-04T20:20:14,569 WARN [IPC Server handler 0 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T20:20:14,569 WARN [IPC Server handler 0 on default port 40063 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T20:20:14,570 WARN [IPC Server handler 0 on default port 40063 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T20:20:14,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741886_1069 (size=18097) 2024-12-04T20:20:14,988 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/bd5d6f5727fc42c3a4b6418a8a05e588 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588 2024-12-04T20:20:14,997 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb4f1c9e49247cbbdbc1a36e5ba2d81f/info of eb4f1c9e49247cbbdbc1a36e5ba2d81f into bd5d6f5727fc42c3a4b6418a8a05e588(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:20:14,997 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:14,997 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f., storeName=eb4f1c9e49247cbbdbc1a36e5ba2d81f/info, priority=13, startTime=1733343614540; duration=0sec 2024-12-04T20:20:14,997 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-12-04T20:20:14,997 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:14,997 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588 because midkey is the same as first or last row 2024-12-04T20:20:14,998 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-12-04T20:20:14,998 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:14,998 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588 because midkey is the same as first or last row 2024-12-04T20:20:14,998 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-12-04T20:20:14,998 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:14,998 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588 because midkey is the same as first or last row 2024-12-04T20:20:14,998 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:20:14,998 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb4f1c9e49247cbbdbc1a36e5ba2d81f:info 2024-12-04T20:20:15,044 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@77f37dae[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741866_1049 to 127.0.0.1:43355 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:15,044 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@79349d20[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741871_1054 to 127.0.0.1:41997 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:15,190 WARN [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-12-04T20:20:15,190 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:15,330 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:15,334 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:15,335 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:15,335 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:15,335 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:20:15,336 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2aaad1eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:15,336 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@719d00c0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:15,424 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5536ce0c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/java.io.tmpdir/jetty-localhost-36267-hadoop-hdfs-3_4_1-tests_jar-_-any-16681034133420405814/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:15,425 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c03d0a2{HTTP/1.1, (http/1.1)}{localhost:36267} 2024-12-04T20:20:15,425 INFO [Time-limited test {}] server.Server(415): Started @127265ms 2024-12-04T20:20:15,426 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:15,492 WARN [Thread-997 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:15,499 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e000e0bbeca42f with lease ID 0xabba01b09ec59521: from storage DS-db585bb6-6935-4414-8f5b-90ce22b682f8 node DatanodeRegistration(127.0.0.1:40161, datanodeUuid=da9387b2-45f0-494c-bf5b-3a8de868daf8, infoPort=40511, infoSecurePort=0, ipcPort=41147, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:15,499 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e000e0bbeca42f with lease ID 0xabba01b09ec59521: from storage DS-b33f0150-0138-452c-8e9b-b120f3f00cc4 node DatanodeRegistration(127.0.0.1:40161, datanodeUuid=da9387b2-45f0-494c-bf5b-3a8de868daf8, infoPort=40511, infoSecurePort=0, ipcPort=41147, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:15,606 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:16,046 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@77f37dae[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741881_1064 to 127.0.0.1:42579 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:16,045 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@79349d20[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741856_1039 to 127.0.0.1:36485 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:16,511 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:17,191 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:17,607 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:18,045 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@79349d20[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45187, datanodeUuid=9d551c21-4bc5-4720-a0b9-ded15e170cf4, infoPort=35487, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741886_1069 to 127.0.0.1:36485 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:18,511 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:19,191 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:19,607 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:20,512 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:21,038 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:20:21,192 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:21,608 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:21,697 ERROR [FSHLog-0-hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData-prefix:c2ef38372881,35483,1733343591060 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:21,697 WARN [FSHLog-0-hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData-prefix:c2ef38372881,35483,1733343591060 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:21,697 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C35483%2C1733343591060:(num 1733343591623) roll requested 2024-12-04T20:20:21,698 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C35483%2C1733343591060.1733343621698 2024-12-04T20:20:21,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:21,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:21,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:21,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:21,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:21,709 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 with entries=54, filesize=26.65 KB; new WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343621698 2024-12-04T20:20:21,709 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:21,709 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:21,709 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 2024-12-04T20:20:21,710 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40511:40511),(127.0.0.1/127.0.0.1:35487:35487)] 2024-12-04T20:20:21,710 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 is not closed yet, will try archiving it next time 2024-12-04T20:20:21,710 WARN [IPC Server handler 3 on default port 40063 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 has not been closed. Lease recovery is in progress. RecoveryId = 1071 for block blk_1073741830_1006 2024-12-04T20:20:21,710 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 after 1ms 2024-12-04T20:20:22,512 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:23,192 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:24,513 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:25,193 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:25,512 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@55b0f970 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1135270363-172.17.0.2-1733343590490:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:43355,null,null]) java.net.ConnectException: Call From c2ef38372881/172.17.0.2 to localhost:44951 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T20:20:25,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741833_1020 (size=455) 2024-12-04T20:20:25,712 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 after 4003ms 2024-12-04T20:20:26,156 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343591936 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs/c2ef38372881%2C34471%2C1733343591112.1733343591936 2024-12-04T20:20:26,159 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343611142 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs/c2ef38372881%2C34471%2C1733343591112.1733343611142 2024-12-04T20:20:26,513 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:27,193 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:27,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741835_1011 (size=393) 2024-12-04T20:20:27,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741833_1020 (size=455) 2024-12-04T20:20:28,500 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@673608d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40161, datanodeUuid=da9387b2-45f0-494c-bf5b-3a8de868daf8, infoPort=40511, infoSecurePort=0, ipcPort=41147, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741831_1007 to 127.0.0.1:36485 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:28,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:20:28,513 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:28,919 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34471%2C1733343591112.1733343628918 2024-12-04T20:20:28,925 WARN [Thread-1031 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1072 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:28,925 WARN [Thread-1031 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741888_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:40161,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:28,925 WARN [Thread-1031 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741888_1072 2024-12-04T20:20:28,926 WARN [Thread-1031 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:28,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:28,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:28,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:28,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:28,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:28,931 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343613165 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343628918 2024-12-04T20:20:28,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40511:40511),(127.0.0.1/127.0.0.1:35487:35487)] 2024-12-04T20:20:28,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741876_1059 (size=12911) 2024-12-04T20:20:28,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343613165 is not closed yet, will try archiving it next time 2024-12-04T20:20:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] regionserver.HRegion(8855): Flush requested on eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:28,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb4f1c9e49247cbbdbc1a36e5ba2d81f 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-04T20:20:28,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/dc7b7625842f4c949e3dff6849f6b169 is 1080, key is row0013/info:/1733343628934/Put/seqid=0 2024-12-04T20:20:28,946 WARN [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36485 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:28,946 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:38746 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741890_1074] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data4]'}, localName='127.0.0.1:40161', datanodeUuid='da9387b2-45f0-494c-bf5b-3a8de868daf8', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741890_1074 to mirror 127.0.0.1:36485 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:28,946 WARN [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40161,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK], DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:28,946 WARN [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741890_1074 2024-12-04T20:20:28,946 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:38746 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741890_1074] {}] datanode.BlockReceiver(316): Block 1073741890 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:28,946 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:38746 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741890_1074] {}] datanode.DataXceiver(331): 127.0.0.1:40161:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38746 dst: /127.0.0.1:40161 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:28,947 WARN [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:28,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741891_1075 (size=8190) 2024-12-04T20:20:28,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741891_1075 (size=8190) 2024-12-04T20:20:28,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/dc7b7625842f4c949e3dff6849f6b169 2024-12-04T20:20:28,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/dc7b7625842f4c949e3dff6849f6b169 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/dc7b7625842f4c949e3dff6849f6b169 2024-12-04T20:20:28,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/dc7b7625842f4c949e3dff6849f6b169, entries=3, sequenceid=66, filesize=8.0 K 2024-12-04T20:20:28,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for eb4f1c9e49247cbbdbc1a36e5ba2d81f in 40ms, sequenceid=66, compaction requested=false 2024-12-04T20:20:28,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:28,977 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K 2024-12-04T20:20:28,977 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:28,977 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588 because midkey is the same as first or last row 2024-12-04T20:20:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] regionserver.HRegion(8855): Flush requested on eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:29,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb4f1c9e49247cbbdbc1a36e5ba2d81f 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB 2024-12-04T20:20:29,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/656d3c0956234c469b8104abdc152318 is 1080, key is row0015/info:/1733343628938/Put/seqid=0 2024-12-04T20:20:29,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741892_1076 (size=14660) 2024-12-04T20:20:29,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741892_1076 (size=14660) 2024-12-04T20:20:29,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/656d3c0956234c469b8104abdc152318 2024-12-04T20:20:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/656d3c0956234c469b8104abdc152318 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/656d3c0956234c469b8104abdc152318 2024-12-04T20:20:29,194 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-12-04T20:20:29,194 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/656d3c0956234c469b8104abdc152318, entries=9, sequenceid=79, filesize=14.3 K 2024-12-04T20:20:29,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for eb4f1c9e49247cbbdbc1a36e5ba2d81f in 36ms, sequenceid=79, compaction requested=true 2024-12-04T20:20:29,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:29,199 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K 2024-12-04T20:20:29,199 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:29,199 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588 because midkey is the same as first or last row 2024-12-04T20:20:29,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb4f1c9e49247cbbdbc1a36e5ba2d81f:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:20:29,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:20:29,199 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:20:29,201 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:20:29,201 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1541): eb4f1c9e49247cbbdbc1a36e5ba2d81f/info is initiating minor compaction (all files) 2024-12-04T20:20:29,201 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb4f1c9e49247cbbdbc1a36e5ba2d81f/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:29,201 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/dc7b7625842f4c949e3dff6849f6b169, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/656d3c0956234c469b8104abdc152318] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp, totalSize=40.0 K 2024-12-04T20:20:29,201 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd5d6f5727fc42c3a4b6418a8a05e588, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733343605136 2024-12-04T20:20:29,202 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc7b7625842f4c949e3dff6849f6b169, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733343615114 2024-12-04T20:20:29,202 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656d3c0956234c469b8104abdc152318, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733343628938 2024-12-04T20:20:29,213 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb4f1c9e49247cbbdbc1a36e5ba2d81f#info#compaction#27 average throughput is 22.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:20:29,214 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/e1b081b87f2e4e6ba739d8f735f7b9a9 is 1080, key is row0002/info:/1733343605136/Put/seqid=0 2024-12-04T20:20:29,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741893_1077 (size=28989) 2024-12-04T20:20:29,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741893_1077 (size=28989) 2024-12-04T20:20:29,226 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/.tmp/info/e1b081b87f2e4e6ba739d8f735f7b9a9 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/e1b081b87f2e4e6ba739d8f735f7b9a9 2024-12-04T20:20:29,234 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb4f1c9e49247cbbdbc1a36e5ba2d81f/info of eb4f1c9e49247cbbdbc1a36e5ba2d81f into e1b081b87f2e4e6ba739d8f735f7b9a9(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:20:29,234 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: 2024-12-04T20:20:29,234 INFO [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f., storeName=eb4f1c9e49247cbbdbc1a36e5ba2d81f/info, priority=13, startTime=1733343629199; duration=0sec 2024-12-04T20:20:29,234 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-12-04T20:20:29,234 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/e1b081b87f2e4e6ba739d8f735f7b9a9 because midkey is the same as first or last row 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/e1b081b87f2e4e6ba739d8f735f7b9a9 because midkey is the same as first or last row 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/e1b081b87f2e4e6ba739d8f735f7b9a9 because midkey is the same as first or last row 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:20:29,235 DEBUG [RS:0;c2ef38372881:34471-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb4f1c9e49247cbbdbc1a36e5ba2d81f:info 2024-12-04T20:20:29,334 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.1733343613165 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs/c2ef38372881%2C34471%2C1733343591112.1733343613165 2024-12-04T20:20:29,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:20:29,364 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:20:29,364 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:20:29,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:20:29,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:20:29,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:20:29,365 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:20:29,366 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=558867555, stopped=false 2024-12-04T20:20:29,366 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,35483,1733343591060 2024-12-04T20:20:29,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:20:29,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:20:29,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:20:29,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:29,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:29,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:29,369 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:20:29,369 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:20:29,370 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:20:29,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:20:29,370 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:20:29,371 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,34471,1733343591112' ***** 2024-12-04T20:20:29,371 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:20:29,371 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,37047,1733343592465' ***** 2024-12-04T20:20:29,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:20:29,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:20:29,371 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:20:29,371 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:20:29,371 INFO [RS:0;c2ef38372881:34471 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:20:29,371 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:20:29,372 INFO [RS:0;c2ef38372881:34471 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:20:29,372 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(3091): Received CLOSE for eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:29,372 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:20:29,372 INFO [RS:1;c2ef38372881:37047 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:20:29,372 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:20:29,372 INFO [RS:1;c2ef38372881:37047 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:20:29,372 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,34471,1733343591112 2024-12-04T20:20:29,372 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,37047,1733343592465 2024-12-04T20:20:29,372 INFO [RS:0;c2ef38372881:34471 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:20:29,372 INFO [RS:1;c2ef38372881:37047 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:20:29,373 INFO [RS:0;c2ef38372881:34471 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:34471. 2024-12-04T20:20:29,373 INFO [RS:1;c2ef38372881:37047 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c2ef38372881:37047. 2024-12-04T20:20:29,373 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb4f1c9e49247cbbdbc1a36e5ba2d81f, disabling compactions & flushes 2024-12-04T20:20:29,373 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:29,373 DEBUG [RS:0;c2ef38372881:34471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:20:29,373 DEBUG [RS:1;c2ef38372881:37047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:20:29,373 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:29,373 DEBUG [RS:0;c2ef38372881:34471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:20:29,373 DEBUG [RS:1;c2ef38372881:37047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:20:29,373 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. after waiting 0 ms 2024-12-04T20:20:29,373 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:29,373 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,37047,1733343592465; all regions closed. 2024-12-04T20:20:29,373 INFO [RS:0;c2ef38372881:34471 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:20:29,373 INFO [RS:0;c2ef38372881:34471 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:20:29,373 INFO [RS:0;c2ef38372881:34471 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:20:29,373 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:20:29,373 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T20:20:29,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,374 DEBUG [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1325): Online Regions={eb4f1c9e49247cbbdbc1a36e5ba2d81f=TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T20:20:29,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,373 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/887f46696b2b4854a4659723968d957c, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/a0054509a5a04ace9a02652f3867bc32, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/446cbf45e98947d38b854508a39240d3, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/3dac2af668c149b38de215c63181733d, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/dc7b7625842f4c949e3dff6849f6b169, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/656d3c0956234c469b8104abdc152318] to archive 2024-12-04T20:20:29,374 DEBUG [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eb4f1c9e49247cbbdbc1a36e5ba2d81f 2024-12-04T20:20:29,374 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:20:29,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,374 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:20:29,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,374 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:20:29,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,374 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:20:29,374 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:20:29,374 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-12-04T20:20:29,374 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,374 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,374 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 2024-12-04T20:20:29,375 ERROR [FSHLog-0-hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d-prefix:c2ef38372881,34471,1733343591112.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,375 WARN [FSHLog-0-hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d-prefix:c2ef38372881,34471,1733343591112.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,375 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C34471%2C1733343591112.meta:.meta(num 1733343592355) roll requested 2024-12-04T20:20:29,375 WARN [IPC Server handler 1 on default port 40063 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 has not been closed. Lease recovery is in progress. RecoveryId = 1078 for block blk_1073741837_1013 2024-12-04T20:20:29,375 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34471%2C1733343591112.meta.1733343629375.meta 2024-12-04T20:20:29,375 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:20:29,375 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 after 1ms 2024-12-04T20:20:29,377 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/887f46696b2b4854a4659723968d957c to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/887f46696b2b4854a4659723968d957c 2024-12-04T20:20:29,378 WARN [Thread-1060 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1079 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,378 WARN [Thread-1060 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741894_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:29,378 WARN [Thread-1060 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741894_1079 2024-12-04T20:20:29,378 WARN [Thread-1060 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:29,379 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/a0054509a5a04ace9a02652f3867bc32 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/a0054509a5a04ace9a02652f3867bc32 2024-12-04T20:20:29,380 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/b3d98c38ce0f42989684347d247e8c45 2024-12-04T20:20:29,382 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/446cbf45e98947d38b854508a39240d3 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/446cbf45e98947d38b854508a39240d3 2024-12-04T20:20:29,383 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/3dac2af668c149b38de215c63181733d to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/3dac2af668c149b38de215c63181733d 2024-12-04T20:20:29,384 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/bd5d6f5727fc42c3a4b6418a8a05e588 2024-12-04T20:20:29,386 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/7d7a0e4e35f24d3daaf79ce25e6c2ff2 2024-12-04T20:20:29,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,387 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/dc7b7625842f4c949e3dff6849f6b169 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/dc7b7625842f4c949e3dff6849f6b169 2024-12-04T20:20:29,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,387 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343629375.meta 2024-12-04T20:20:29,388 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,388 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43355,DS-0413b787-b062-40fb-aefd-17b90257faf5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,388 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta 2024-12-04T20:20:29,388 WARN [IPC Server handler 0 on default port 40063 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta has not been closed. Lease recovery is in progress. RecoveryId = 1081 for block blk_1073741834_1010 2024-12-04T20:20:29,388 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta after 0ms 2024-12-04T20:20:29,388 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/656d3c0956234c469b8104abdc152318 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/info/656d3c0956234c469b8104abdc152318 2024-12-04T20:20:29,389 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c2ef38372881:35483 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T20:20:29,389 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [887f46696b2b4854a4659723968d957c=10347, a0054509a5a04ace9a02652f3867bc32=12506, b3d98c38ce0f42989684347d247e8c45=17994, 446cbf45e98947d38b854508a39240d3=6027, 3dac2af668c149b38de215c63181733d=6027, bd5d6f5727fc42c3a4b6418a8a05e588=18097, 7d7a0e4e35f24d3daaf79ce25e6c2ff2=6027, dc7b7625842f4c949e3dff6849f6b169=8190, 656d3c0956234c469b8104abdc152318=14660] 2024-12-04T20:20:29,393 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35487:35487),(127.0.0.1/127.0.0.1:40511:40511)] 2024-12-04T20:20:29,393 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta is not closed yet, will try archiving it next time 2024-12-04T20:20:29,396 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/default/TestLogRolling-testLogRollOnDatanodeDeath/eb4f1c9e49247cbbdbc1a36e5ba2d81f/recovered.edits/83.seqid, newMaxSeqId=83, maxSeqId=1 2024-12-04T20:20:29,397 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:29,397 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb4f1c9e49247cbbdbc1a36e5ba2d81f: Waiting for close lock at 1733343629372Running coprocessor pre-close hooks at 1733343629372Disabling compacts and flushes for region at 1733343629372Disabling writes for close at 1733343629373 (+1 ms)Writing region close event to WAL at 1733343629392 (+19 ms)Running coprocessor post-close hooks at 1733343629397 (+5 ms)Closed at 1733343629397 2024-12-04T20:20:29,397 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f. 2024-12-04T20:20:29,407 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/info/e184c35c3527475ebd628fea0b159b36 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733343592536.eb4f1c9e49247cbbdbc1a36e5ba2d81f./info:regioninfo/1733343592913/Put/seqid=0 2024-12-04T20:20:29,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741896_1082 (size=7089) 2024-12-04T20:20:29,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741896_1082 (size=7089) 2024-12-04T20:20:29,413 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/info/e184c35c3527475ebd628fea0b159b36 2024-12-04T20:20:29,431 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/ns/cf0d84efa72e41659a1f6cf962f28017 is 43, key is default/ns:d/1733343592399/Put/seqid=0 2024-12-04T20:20:29,433 WARN [Thread-1073 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1083 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,433 WARN [Thread-1073 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741897_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK], DatanodeInfoWithStorage[127.0.0.1:40161,DS-db585bb6-6935-4414-8f5b-90ce22b682f8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:29,433 WARN [Thread-1073 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741897_1083 2024-12-04T20:20:29,434 WARN [Thread-1073 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:29,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741898_1084 (size=5153) 2024-12-04T20:20:29,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741898_1084 (size=5153) 2024-12-04T20:20:29,439 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/ns/cf0d84efa72e41659a1f6cf962f28017 2024-12-04T20:20:29,459 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/table/ac8627c36f0c426baca3910ba81b47c0 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733343592926/Put/seqid=0 2024-12-04T20:20:29,462 WARN [Thread-1079 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1085 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36485 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:29,462 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:53762 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741899_1085] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10]'}, localName='127.0.0.1:45187', datanodeUuid='9d551c21-4bc5-4720-a0b9-ded15e170cf4', xmitsInProgress=0}:Exception transferring block BP-1135270363-172.17.0.2-1733343590490:blk_1073741899_1085 to mirror 127.0.0.1:36485 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:29,462 WARN [Thread-1079 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1135270363-172.17.0.2-1733343590490:blk_1073741899_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45187,DS-0ab71893-c5fb-41d0-ad81-a790574e6733,DISK], DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK]) is bad. 2024-12-04T20:20:29,462 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:53762 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741899_1085] {}] datanode.BlockReceiver(316): Block 1073741899 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T20:20:29,462 WARN [Thread-1079 {}] hdfs.DataStreamer(1850): Abandoning BP-1135270363-172.17.0.2-1733343590490:blk_1073741899_1085 2024-12-04T20:20:29,462 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_302237595_22 at /127.0.0.1:53762 [Receiving block BP-1135270363-172.17.0.2-1733343590490:blk_1073741899_1085] {}] datanode.DataXceiver(331): 127.0.0.1:45187:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53762 dst: /127.0.0.1:45187 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:29,462 WARN [Thread-1079 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36485,DS-f865e4f1-614b-4edc-95fa-52adbbec3fc8,DISK] 2024-12-04T20:20:29,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741900_1086 (size=5424) 2024-12-04T20:20:29,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741900_1086 (size=5424) 2024-12-04T20:20:29,467 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/table/ac8627c36f0c426baca3910ba81b47c0 2024-12-04T20:20:29,474 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/info/e184c35c3527475ebd628fea0b159b36 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/info/e184c35c3527475ebd628fea0b159b36 2024-12-04T20:20:29,480 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/info/e184c35c3527475ebd628fea0b159b36, entries=10, sequenceid=11, filesize=6.9 K 2024-12-04T20:20:29,481 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/ns/cf0d84efa72e41659a1f6cf962f28017 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/ns/cf0d84efa72e41659a1f6cf962f28017 2024-12-04T20:20:29,486 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/ns/cf0d84efa72e41659a1f6cf962f28017, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T20:20:29,488 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/.tmp/table/ac8627c36f0c426baca3910ba81b47c0 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/table/ac8627c36f0c426baca3910ba81b47c0 2024-12-04T20:20:29,494 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/table/ac8627c36f0c426baca3910ba81b47c0, entries=2, sequenceid=11, filesize=5.3 K 2024-12-04T20:20:29,495 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 121ms, sequenceid=11, compaction requested=false 2024-12-04T20:20:29,500 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T20:20:29,500 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:20:29,500 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:20:29,500 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343629374Running coprocessor pre-close hooks at 1733343629374Disabling compacts and flushes for region at 1733343629374Disabling writes for close at 1733343629374Obtaining lock to block concurrent updates at 1733343629374Preparing flush snapshotting stores in 1588230740 at 1733343629374Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733343629375 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733343629393 (+18 ms)Flushing 1588230740/info: creating writer at 1733343629394 (+1 ms)Flushing 1588230740/info: appending metadata at 1733343629407 (+13 ms)Flushing 1588230740/info: closing flushed file at 1733343629407Flushing 1588230740/ns: creating writer at 1733343629419 (+12 ms)Flushing 1588230740/ns: appending metadata at 1733343629431 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733343629431Flushing 1588230740/table: creating writer at 1733343629445 (+14 ms)Flushing 1588230740/table: appending metadata at 1733343629458 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733343629458Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39a0ea35: reopening flushed file at 1733343629473 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e540896: reopening flushed file at 1733343629480 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65ebac63: reopening flushed file at 1733343629487 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 121ms, sequenceid=11, compaction requested=false at 1733343629495 (+8 ms)Writing region close event to WAL at 1733343629496 (+1 ms)Running coprocessor post-close hooks at 1733343629500 (+4 ms)Closed at 1733343629500 2024-12-04T20:20:29,501 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:20:29,506 INFO [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T20:20:29,507 INFO [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T20:20:29,574 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,34471,1733343591112; all regions closed. 2024-12-04T20:20:29,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:29,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741895_1080 (size=825) 2024-12-04T20:20:29,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741895_1080 (size=825) 2024-12-04T20:20:29,804 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:20:29,861 INFO [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T20:20:29,861 INFO [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T20:20:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741876_1059 (size=12911) 2024-12-04T20:20:30,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741836_1012 (size=76) 2024-12-04T20:20:30,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:20:30,509 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:20:31,501 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@673608d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40161, datanodeUuid=da9387b2-45f0-494c-bf5b-3a8de868daf8, infoPort=40511, infoSecurePort=0, ipcPort=41147, storageInfo=lv=-57;cid=testClusterID;nsid=248274260;c=1733343590490):Failed to transfer BP-1135270363-172.17.0.2-1733343590490:blk_1073741828_1004 to 127.0.0.1:36485 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:31,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:20:32,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T20:20:32,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:20:32,621 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:20:32,722 INFO [master/c2ef38372881:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T20:20:32,722 INFO [master/c2ef38372881:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T20:20:33,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741839_1021 (size=2431) 2024-12-04T20:20:33,377 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 after 4003ms 2024-12-04T20:20:33,390 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta after 4002ms 2024-12-04T20:20:33,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:20:33,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:20:34,375 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-04T20:20:34,377 DEBUG [RS:1;c2ef38372881:37047 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs 2024-12-04T20:20:34,377 INFO [RS:1;c2ef38372881:37047 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C37047%2C1733343592465:(num 1733343592641) 2024-12-04T20:20:34,377 DEBUG [RS:1;c2ef38372881:37047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:20:34,377 INFO [RS:1;c2ef38372881:37047 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:20:34,378 INFO [RS:1;c2ef38372881:37047 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:20:34,378 INFO [RS:1;c2ef38372881:37047 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T20:20:34,378 INFO [RS:1;c2ef38372881:37047 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:20:34,378 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:20:34,378 INFO [RS:1;c2ef38372881:37047 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:20:34,378 INFO [RS:1;c2ef38372881:37047 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:20:34,378 INFO [RS:1;c2ef38372881:37047 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:20:34,378 INFO [RS:1;c2ef38372881:37047 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37047 2024-12-04T20:20:34,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,37047,1733343592465 2024-12-04T20:20:34,380 INFO [RS:1;c2ef38372881:37047 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:20:34,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:20:34,382 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,37047,1733343592465] 2024-12-04T20:20:34,383 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,37047,1733343592465 already deleted, retry=false 2024-12-04T20:20:34,383 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,37047,1733343592465 expired; onlineServers=1 2024-12-04T20:20:34,384 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:34,399 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,413 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,414 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,414 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,414 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,415 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,420 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,420 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:20:34,482 INFO [RS:1;c2ef38372881:37047 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:20:34,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37047-0x100a6e396ef0002, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:20:34,482 INFO [RS:1;c2ef38372881:37047 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,37047,1733343592465; zookeeper connection closed. 2024-12-04T20:20:34,482 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3be266de {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3be266de 2024-12-04T20:20:34,576 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-04T20:20:34,584 DEBUG [RS:0;c2ef38372881:34471 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs 2024-12-04T20:20:34,584 INFO [RS:0;c2ef38372881:34471 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C34471%2C1733343591112.meta:.meta(num 1733343629375) 2024-12-04T20:20:34,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741889_1073 (size=16308) 2024-12-04T20:20:34,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741889_1073 (size=16308) 2024-12-04T20:20:34,590 DEBUG [RS:0;c2ef38372881:34471 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs 2024-12-04T20:20:34,590 INFO [RS:0;c2ef38372881:34471 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C34471%2C1733343591112:(num 1733343628918) 2024-12-04T20:20:34,590 DEBUG [RS:0;c2ef38372881:34471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:20:34,590 INFO [RS:0;c2ef38372881:34471 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:20:34,590 INFO [RS:0;c2ef38372881:34471 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:20:34,590 INFO [RS:0;c2ef38372881:34471 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T20:20:34,590 INFO [RS:0;c2ef38372881:34471 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:20:34,590 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:20:34,590 INFO [RS:0;c2ef38372881:34471 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34471 2024-12-04T20:20:34,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,34471,1733343591112 2024-12-04T20:20:34,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:20:34,592 INFO [RS:0;c2ef38372881:34471 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:20:34,593 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,34471,1733343591112] 2024-12-04T20:20:34,593 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,34471,1733343591112 already deleted, retry=false 2024-12-04T20:20:34,594 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,34471,1733343591112 expired; onlineServers=0 2024-12-04T20:20:34,594 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,35483,1733343591060' ***** 2024-12-04T20:20:34,594 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:20:34,594 INFO [M:0;c2ef38372881:35483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:20:34,594 INFO [M:0;c2ef38372881:35483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:20:34,594 DEBUG [M:0;c2ef38372881:35483 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:20:34,594 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:20:34,594 DEBUG [M:0;c2ef38372881:35483 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:20:34,594 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343591696 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343591696,5,FailOnTimeoutGroup] 2024-12-04T20:20:34,594 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343591696 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343591696,5,FailOnTimeoutGroup] 2024-12-04T20:20:34,594 INFO [M:0;c2ef38372881:35483 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:20:34,594 INFO [M:0;c2ef38372881:35483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:20:34,595 DEBUG [M:0;c2ef38372881:35483 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:20:34,595 INFO [M:0;c2ef38372881:35483 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:20:34,595 INFO [M:0;c2ef38372881:35483 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:20:34,595 INFO [M:0;c2ef38372881:35483 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:20:34,595 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:20:34,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:20:34,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:34,595 DEBUG [M:0;c2ef38372881:35483 {}] zookeeper.ZKUtil(347): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:20:34,595 WARN [M:0;c2ef38372881:35483 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:20:34,596 INFO [M:0;c2ef38372881:35483 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/.lastflushedseqids 2024-12-04T20:20:34,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741901_1087 (size=130) 2024-12-04T20:20:34,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741901_1087 (size=130) 2024-12-04T20:20:34,603 INFO [M:0;c2ef38372881:35483 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:20:34,603 INFO [M:0;c2ef38372881:35483 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:20:34,603 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:20:34,603 INFO [M:0;c2ef38372881:35483 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:20:34,603 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:20:34,603 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:20:34,603 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:20:34,603 INFO [M:0;c2ef38372881:35483 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.24 KB heapSize=29.47 KB 2024-12-04T20:20:34,616 DEBUG [M:0;c2ef38372881:35483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/964ced02d8434762b9ddfb6ec0dc22c4 is 82, key is hbase:meta,,1/info:regioninfo/1733343592387/Put/seqid=0 2024-12-04T20:20:34,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741902_1088 (size=5672) 2024-12-04T20:20:34,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741902_1088 (size=5672) 2024-12-04T20:20:34,622 INFO [M:0;c2ef38372881:35483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/964ced02d8434762b9ddfb6ec0dc22c4 2024-12-04T20:20:34,639 DEBUG [M:0;c2ef38372881:35483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21527d87efb94a72896d8e0634e06028 is 773, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733343592932/Put/seqid=0 2024-12-04T20:20:34,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741903_1089 (size=6254) 2024-12-04T20:20:34,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741903_1089 (size=6254) 2024-12-04T20:20:34,644 INFO [M:0;c2ef38372881:35483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21527d87efb94a72896d8e0634e06028 2024-12-04T20:20:34,650 INFO [M:0;c2ef38372881:35483 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21527d87efb94a72896d8e0634e06028 2024-12-04T20:20:34,662 DEBUG [M:0;c2ef38372881:35483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6ae574e46104cc0af6e4f78b16659af is 69, key is c2ef38372881,34471,1733343591112/rs:state/1733343591781/Put/seqid=0 2024-12-04T20:20:34,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741904_1090 (size=5224) 2024-12-04T20:20:34,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741904_1090 (size=5224) 2024-12-04T20:20:34,668 INFO [M:0;c2ef38372881:35483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6ae574e46104cc0af6e4f78b16659af 2024-12-04T20:20:34,690 DEBUG [M:0;c2ef38372881:35483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6acfa74ff6d9497c9aadba99494ce750 is 52, key is load_balancer_on/state:d/1733343592450/Put/seqid=0 2024-12-04T20:20:34,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:20:34,693 INFO [RS:0;c2ef38372881:34471 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:20:34,693 INFO [RS:0;c2ef38372881:34471 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,34471,1733343591112; zookeeper connection closed. 2024-12-04T20:20:34,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34471-0x100a6e396ef0001, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:20:34,693 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@65d4b1fd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@65d4b1fd 2024-12-04T20:20:34,694 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-04T20:20:34,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741905_1091 (size=5056) 2024-12-04T20:20:34,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741905_1091 (size=5056) 2024-12-04T20:20:34,696 INFO [M:0;c2ef38372881:35483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6acfa74ff6d9497c9aadba99494ce750 2024-12-04T20:20:34,701 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/964ced02d8434762b9ddfb6ec0dc22c4 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/964ced02d8434762b9ddfb6ec0dc22c4 2024-12-04T20:20:34,706 INFO [M:0;c2ef38372881:35483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/964ced02d8434762b9ddfb6ec0dc22c4, entries=8, sequenceid=60, filesize=5.5 K 2024-12-04T20:20:34,707 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21527d87efb94a72896d8e0634e06028 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21527d87efb94a72896d8e0634e06028 2024-12-04T20:20:34,712 INFO [M:0;c2ef38372881:35483 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21527d87efb94a72896d8e0634e06028 2024-12-04T20:20:34,712 INFO [M:0;c2ef38372881:35483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21527d87efb94a72896d8e0634e06028, entries=6, sequenceid=60, filesize=6.1 K 2024-12-04T20:20:34,714 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6ae574e46104cc0af6e4f78b16659af as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c6ae574e46104cc0af6e4f78b16659af 2024-12-04T20:20:34,721 INFO [M:0;c2ef38372881:35483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c6ae574e46104cc0af6e4f78b16659af, entries=2, sequenceid=60, filesize=5.1 K 2024-12-04T20:20:34,722 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6acfa74ff6d9497c9aadba99494ce750 as hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6acfa74ff6d9497c9aadba99494ce750 2024-12-04T20:20:34,728 INFO [M:0;c2ef38372881:35483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6acfa74ff6d9497c9aadba99494ce750, entries=1, sequenceid=60, filesize=4.9 K 2024-12-04T20:20:34,729 INFO [M:0;c2ef38372881:35483 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.24 KB/23793, heapSize ~29.41 KB/30112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 125ms, sequenceid=60, compaction requested=false 2024-12-04T20:20:34,730 INFO [M:0;c2ef38372881:35483 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:20:34,730 DEBUG [M:0;c2ef38372881:35483 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343634603Disabling compacts and flushes for region at 1733343634603Disabling writes for close at 1733343634603Obtaining lock to block concurrent updates at 1733343634603Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733343634603Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23793, getHeapSize=30112, getOffHeapSize=0, getCellsCount=71 at 1733343634603Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733343634604 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733343634604Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733343634616 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733343634616Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733343634626 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733343634639 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733343634639Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733343634650 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733343634662 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733343634662Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733343634673 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733343634690 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733343634690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c802a06: reopening flushed file at 1733343634700 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f7a1632: reopening flushed file at 1733343634706 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42c9cea2: reopening flushed file at 1733343634712 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7faa9a81: reopening flushed file at 1733343634721 (+9 ms)Finished flush of dataSize ~23.24 KB/23793, heapSize ~29.41 KB/30112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 125ms, sequenceid=60, compaction requested=false at 1733343634729 (+8 ms)Writing region close event to WAL at 1733343634730 (+1 ms)Closed at 1733343634730 2024-12-04T20:20:34,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45187 is added to blk_1073741887_1070 (size=1045) 2024-12-04T20:20:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40161 is added to blk_1073741887_1070 (size=1045) 2024-12-04T20:20:34,923 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T20:20:34,937 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,938 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,938 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,939 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,939 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,940 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,943 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:34,945 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:35,385 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:35,391 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:35,516 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1fc3c43a {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1135270363-172.17.0.2-1733343590490:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:43355,null,null]) java.net.ConnectException: Call From c2ef38372881/172.17.0.2 to localhost:44951 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T20:20:35,723 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/WALs/c2ef38372881,35483,1733343591060/c2ef38372881%2C35483%2C1733343591060.1733343591623 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/oldWALs/c2ef38372881%2C35483%2C1733343591060.1733343591623 2024-12-04T20:20:35,731 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/MasterData/oldWALs/c2ef38372881%2C35483%2C1733343591060.1733343591623 to hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/oldWALs/c2ef38372881%2C35483%2C1733343591060.1733343591623$masterlocalwal$ 2024-12-04T20:20:35,732 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:20:35,732 INFO [M:0;c2ef38372881:35483 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:20:35,732 INFO [M:0;c2ef38372881:35483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35483 2024-12-04T20:20:35,732 INFO [M:0;c2ef38372881:35483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:20:35,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:20:35,834 INFO [M:0;c2ef38372881:35483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:20:35,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35483-0x100a6e396ef0000, quorum=127.0.0.1:63536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:20:35,841 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5536ce0c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:35,842 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c03d0a2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:35,842 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:35,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@719d00c0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:35,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2aaad1eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:35,845 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:35,845 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:35,846 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:35,846 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1135270363-172.17.0.2-1733343590490 (Datanode Uuid da9387b2-45f0-494c-bf5b-3a8de868daf8) service to localhost/127.0.0.1:40063 2024-12-04T20:20:35,845 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@359f063f {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:43355,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:44951 , LocalHost:localPort c2ef38372881/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T20:20:35,846 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@359f063f {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:40161,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1135270363-172.17.0.2-1733343590490 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:35,846 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@359f063f {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:43355,null,null], DatanodeInfoWithStorage[127.0.0.1:40161,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1135270363-172.17.0.2-1733343590490:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:43355,null,null], DatanodeInfoWithStorage[127.0.0.1:40161,null,null]] 2024-12-04T20:20:35,846 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@359f063f {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:43355,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1135270363-172.17.0.2-1733343590490 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:35,846 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@359f063f {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:40161,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1135270363-172.17.0.2-1733343590490 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:35,846 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@359f063f {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:43355,null,null], DatanodeInfoWithStorage[127.0.0.1:40161,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1135270363-172.17.0.2-1733343590490:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:43355,null,null], DatanodeInfoWithStorage[127.0.0.1:40161,null,null]] 2024-12-04T20:20:35,846 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data3/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:35,847 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data4/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:35,847 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:35,848 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28246fba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:35,849 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5056747b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:35,849 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:35,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68004957{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:35,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dc59954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:35,850 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:35,850 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:35,850 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:35,850 WARN [BP-1135270363-172.17.0.2-1733343590490 heartbeating to localhost/127.0.0.1:40063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1135270363-172.17.0.2-1733343590490 (Datanode Uuid 9d551c21-4bc5-4720-a0b9-ded15e170cf4) service to localhost/127.0.0.1:40063 2024-12-04T20:20:35,850 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data9/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:35,851 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/cluster_23f7c998-c795-bfeb-9de4-2a705d719c39/data/data10/current/BP-1135270363-172.17.0.2-1733343590490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:35,851 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:35,855 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@195100a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:20:35,856 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e349773{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:35,856 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:35,856 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a488aac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:35,856 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2305029e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:35,864 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:20:35,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T20:20:35,901 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=156 (was 82) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45475 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40063 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40063 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40063 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-19-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40063 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40063 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-18-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40063 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40063 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f9078bf52a8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-19-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40063 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-19-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40063 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:45475 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40063 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-18-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f9078bf52a8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:40063 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-18-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=32 (was 52), ProcessCount=11 (was 11), AvailableMemoryMB=4614 (was 5380) 2024-12-04T20:20:35,908 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=156, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=32, ProcessCount=11, AvailableMemoryMB=4614 2024-12-04T20:20:35,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:20:35,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.log.dir so I do NOT create it in target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb 2024-12-04T20:20:35,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/daf29109-2059-7bac-579c-5c9f006c8ed4/hadoop.tmp.dir so I do NOT create it in target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb 2024-12-04T20:20:35,908 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d, deleteOnExit=true 2024-12-04T20:20:35,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:20:35,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/test.cache.data in system properties and HBase conf 2024-12-04T20:20:35,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:20:35,909 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:20:35,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:20:35,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:20:35,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:20:35,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:20:35,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:20:35,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:20:35,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:20:35,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:20:35,920 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:20:35,970 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:35,975 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:35,985 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:35,985 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:35,985 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:20:35,986 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:35,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@421a8f73{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:35,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13fdd007{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:36,075 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1ea36316{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/java.io.tmpdir/jetty-localhost-43867-hadoop-hdfs-3_4_1-tests_jar-_-any-10024333822986964523/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:20:36,075 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d48810f{HTTP/1.1, (http/1.1)}{localhost:43867} 2024-12-04T20:20:36,075 INFO [Time-limited test {}] server.Server(415): Started @147915ms 2024-12-04T20:20:36,086 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:20:36,136 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:36,139 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:36,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:36,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:36,141 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:20:36,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@66c0323e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:36,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74ea1d44{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:36,230 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6e0e18a9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/java.io.tmpdir/jetty-localhost-46789-hadoop-hdfs-3_4_1-tests_jar-_-any-12468107654354859465/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:36,230 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@a5d6d97{HTTP/1.1, (http/1.1)}{localhost:46789} 2024-12-04T20:20:36,231 INFO [Time-limited test {}] server.Server(415): Started @148071ms 2024-12-04T20:20:36,232 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:36,275 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:36,278 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:36,279 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:36,279 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:36,279 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:20:36,279 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28441b3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:36,280 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4360f0f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:36,301 WARN [Thread-1203 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data1/current/BP-1711332990-172.17.0.2-1733343635930/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:36,301 WARN [Thread-1204 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data2/current/BP-1711332990-172.17.0.2-1733343635930/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:36,314 WARN [Thread-1182 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:36,317 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2864dd67ec996a69 with lease ID 0xafa9e5de11572c3: Processing first storage report for DS-1e373622-0db0-4fff-8d45-0a62e726ebcf from datanode DatanodeRegistration(127.0.0.1:34201, datanodeUuid=81123f1c-02df-4806-bcc6-1ecb87f7a5bf, infoPort=45143, infoSecurePort=0, ipcPort=37989, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930) 2024-12-04T20:20:36,317 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2864dd67ec996a69 with lease ID 0xafa9e5de11572c3: from storage DS-1e373622-0db0-4fff-8d45-0a62e726ebcf node DatanodeRegistration(127.0.0.1:34201, datanodeUuid=81123f1c-02df-4806-bcc6-1ecb87f7a5bf, infoPort=45143, infoSecurePort=0, ipcPort=37989, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T20:20:36,317 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2864dd67ec996a69 with lease ID 0xafa9e5de11572c3: Processing first storage report for DS-947c3f6e-224f-479e-bca0-5a3efadfdd8d from datanode DatanodeRegistration(127.0.0.1:34201, datanodeUuid=81123f1c-02df-4806-bcc6-1ecb87f7a5bf, infoPort=45143, infoSecurePort=0, ipcPort=37989, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930) 2024-12-04T20:20:36,317 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2864dd67ec996a69 with lease ID 0xafa9e5de11572c3: from storage DS-947c3f6e-224f-479e-bca0-5a3efadfdd8d node DatanodeRegistration(127.0.0.1:34201, datanodeUuid=81123f1c-02df-4806-bcc6-1ecb87f7a5bf, infoPort=45143, infoSecurePort=0, ipcPort=37989, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:36,371 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9982f0a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/java.io.tmpdir/jetty-localhost-44245-hadoop-hdfs-3_4_1-tests_jar-_-any-2980477741864792768/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:36,371 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70df7796{HTTP/1.1, (http/1.1)}{localhost:44245} 2024-12-04T20:20:36,371 INFO [Time-limited test {}] server.Server(415): Started @148211ms 2024-12-04T20:20:36,372 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:36,386 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:36,392 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:36,423 WARN [Thread-1229 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data3/current/BP-1711332990-172.17.0.2-1733343635930/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:36,423 WARN [Thread-1230 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data4/current/BP-1711332990-172.17.0.2-1733343635930/current, will proceed with Du for space computation calculation, 2024-12-04T20:20:36,438 WARN [Thread-1218 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:36,440 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x118e5d68d408bcbc with lease ID 0xafa9e5de11572c4: Processing first storage report for DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3 from datanode DatanodeRegistration(127.0.0.1:46497, datanodeUuid=9f6ab646-fabd-46cc-b732-79c4649a8ae1, infoPort=42877, infoSecurePort=0, ipcPort=38227, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930) 2024-12-04T20:20:36,440 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x118e5d68d408bcbc with lease ID 0xafa9e5de11572c4: from storage DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3 node DatanodeRegistration(127.0.0.1:46497, datanodeUuid=9f6ab646-fabd-46cc-b732-79c4649a8ae1, infoPort=42877, infoSecurePort=0, ipcPort=38227, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:36,440 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x118e5d68d408bcbc with lease ID 0xafa9e5de11572c4: Processing first storage report for DS-5c47abdd-12e0-479a-9821-69d4b9e569a6 from datanode DatanodeRegistration(127.0.0.1:46497, datanodeUuid=9f6ab646-fabd-46cc-b732-79c4649a8ae1, infoPort=42877, infoSecurePort=0, ipcPort=38227, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930) 2024-12-04T20:20:36,440 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x118e5d68d408bcbc with lease ID 0xafa9e5de11572c4: from storage DS-5c47abdd-12e0-479a-9821-69d4b9e569a6 node DatanodeRegistration(127.0.0.1:46497, datanodeUuid=9f6ab646-fabd-46cc-b732-79c4649a8ae1, infoPort=42877, infoSecurePort=0, ipcPort=38227, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:36,494 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb 2024-12-04T20:20:36,500 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/zookeeper_0, clientPort=50038, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:20:36,501 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50038 2024-12-04T20:20:36,502 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:20:36,504 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:20:36,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:20:36,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:20:36,516 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28 with version=8 2024-12-04T20:20:36,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase-staging 2024-12-04T20:20:36,518 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:20:36,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:20:36,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:20:36,518 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:20:36,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:20:36,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:20:36,518 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:20:36,518 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:20:36,519 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33473 2024-12-04T20:20:36,520 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33473 connecting to ZooKeeper ensemble=127.0.0.1:50038 2024-12-04T20:20:36,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:334730x0, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:20:36,524 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33473-0x100a6e448800000 connected 2024-12-04T20:20:36,536 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:20:36,537 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:20:36,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:20:36,539 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28, hbase.cluster.distributed=false 2024-12-04T20:20:36,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:20:36,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33473 2024-12-04T20:20:36,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33473 2024-12-04T20:20:36,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33473 2024-12-04T20:20:36,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33473 2024-12-04T20:20:36,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33473 2024-12-04T20:20:36,559 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:20:36,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:20:36,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:20:36,559 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:20:36,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:20:36,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:20:36,559 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:20:36,559 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:20:36,560 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40035 2024-12-04T20:20:36,561 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40035 connecting to ZooKeeper ensemble=127.0.0.1:50038 2024-12-04T20:20:36,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:20:36,563 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:20:36,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400350x0, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:20:36,566 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40035-0x100a6e448800001 connected 2024-12-04T20:20:36,566 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:20:36,567 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:20:36,567 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:20:36,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:20:36,568 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:20:36,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40035 2024-12-04T20:20:36,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40035 2024-12-04T20:20:36,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40035 2024-12-04T20:20:36,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40035 2024-12-04T20:20:36,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40035 2024-12-04T20:20:36,580 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:33473 2024-12-04T20:20:36,580 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,33473,1733343636517 2024-12-04T20:20:36,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:20:36,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:20:36,582 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,33473,1733343636517 2024-12-04T20:20:36,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:20:36,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,583 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:20:36,583 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,33473,1733343636517 from backup master directory 2024-12-04T20:20:36,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:20:36,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,33473,1733343636517 2024-12-04T20:20:36,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:20:36,584 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:20:36,584 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,33473,1733343636517 2024-12-04T20:20:36,590 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/hbase.id] with ID: 2725106b-318e-4384-b29c-c4c35fb6a122 2024-12-04T20:20:36,591 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/.tmp/hbase.id 2024-12-04T20:20:36,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:20:36,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:20:36,598 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/.tmp/hbase.id]:[hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/hbase.id] 2024-12-04T20:20:36,610 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:20:36,610 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:20:36,612 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-04T20:20:36,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:20:36,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:20:36,620 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:20:36,620 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:20:36,621 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:20:36,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:20:36,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:20:36,629 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store 2024-12-04T20:20:36,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:20:36,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:20:36,636 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:20:36,636 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:20:36,636 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:20:36,636 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:20:36,636 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:20:36,636 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:20:36,636 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:20:36,636 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343636636Disabling compacts and flushes for region at 1733343636636Disabling writes for close at 1733343636636Writing region close event to WAL at 1733343636636Closed at 1733343636636 2024-12-04T20:20:36,637 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/.initializing 2024-12-04T20:20:36,637 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517 2024-12-04T20:20:36,639 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C33473%2C1733343636517, suffix=, logDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517, archiveDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/oldWALs, maxLogs=10 2024-12-04T20:20:36,639 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33473%2C1733343636517.1733343636639 2024-12-04T20:20:36,644 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 2024-12-04T20:20:36,649 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42877:42877),(127.0.0.1/127.0.0.1:45143:45143)] 2024-12-04T20:20:36,649 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:20:36,650 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:20:36,650 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,650 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:20:36,653 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:36,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:20:36,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:20:36,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:20:36,658 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:20:36,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:20:36,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:20:36,660 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,661 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,661 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,663 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,663 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,663 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:20:36,664 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:20:36,666 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:20:36,667 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=755507, jitterRate=-0.039324402809143066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:20:36,667 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733343636650Initializing all the Stores at 1733343636651 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343636651Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343636651Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343636651Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343636651Cleaning up temporary data from old regions at 1733343636663 (+12 ms)Region opened successfully at 1733343636667 (+4 ms) 2024-12-04T20:20:36,667 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:20:36,671 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ebbcef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:20:36,672 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:20:36,672 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:20:36,672 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:20:36,673 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:20:36,673 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T20:20:36,674 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T20:20:36,674 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:20:36,676 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:20:36,677 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:20:36,678 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:20:36,678 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:20:36,679 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:20:36,680 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:20:36,681 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:20:36,682 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:20:36,682 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:20:36,684 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:20:36,684 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:20:36,687 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:20:36,687 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:20:36,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:20:36,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:20:36,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,689 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,33473,1733343636517, sessionid=0x100a6e448800000, setting cluster-up flag (Was=false) 2024-12-04T20:20:36,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:20:36,693 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,33473,1733343636517 2024-12-04T20:20:36,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:36,698 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:20:36,699 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,33473,1733343636517 2024-12-04T20:20:36,700 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:20:36,701 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:20:36,702 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:20:36,702 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:20:36,702 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,33473,1733343636517 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:20:36,703 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:20:36,703 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:20:36,703 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:20:36,703 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:20:36,703 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:20:36,703 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,703 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:20:36,704 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,704 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733343666704 2024-12-04T20:20:36,704 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:20:36,704 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:20:36,705 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:20:36,705 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:20:36,705 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:20:36,705 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:20:36,705 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,705 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:20:36,705 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:20:36,705 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:20:36,705 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:20:36,705 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:20:36,706 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:20:36,706 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:20:36,706 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343636706,5,FailOnTimeoutGroup] 2024-12-04T20:20:36,706 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343636706,5,FailOnTimeoutGroup] 2024-12-04T20:20:36,706 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,706 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:20:36,706 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,706 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,706 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,706 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:20:36,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:20:36,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:20:36,717 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:20:36,717 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28 2024-12-04T20:20:36,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:20:36,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:20:36,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:20:36,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:20:36,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:20:36,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:36,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:20:36,737 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:20:36,737 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:36,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:20:36,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:20:36,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:36,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:20:36,741 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:20:36,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:36,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:36,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:20:36,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740 2024-12-04T20:20:36,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740 2024-12-04T20:20:36,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:20:36,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:20:36,744 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:20:36,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:20:36,747 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:20:36,747 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=798088, jitterRate=0.014822587370872498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:20:36,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733343636731Initializing all the Stores at 1733343636732 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343636732Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343636733 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343636733Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343636733Cleaning up temporary data from old regions at 1733343636744 (+11 ms)Region opened successfully at 1733343636748 (+4 ms) 2024-12-04T20:20:36,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:20:36,748 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:20:36,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:20:36,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:20:36,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:20:36,748 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:20:36,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343636748Disabling compacts and flushes for region at 1733343636748Disabling writes for close at 1733343636748Writing region close event to WAL at 1733343636748Closed at 1733343636748 2024-12-04T20:20:36,750 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:20:36,750 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:20:36,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:20:36,751 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:20:36,752 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:20:36,771 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(746): ClusterId : 2725106b-318e-4384-b29c-c4c35fb6a122 2024-12-04T20:20:36,771 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:20:36,773 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:20:36,773 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:20:36,775 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:20:36,775 DEBUG [RS:0;c2ef38372881:40035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6665ad32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:20:36,785 DEBUG [RS:0;c2ef38372881:40035 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:40035 2024-12-04T20:20:36,785 INFO [RS:0;c2ef38372881:40035 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:20:36,785 INFO [RS:0;c2ef38372881:40035 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:20:36,785 DEBUG [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:20:36,786 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,33473,1733343636517 with port=40035, startcode=1733343636559 2024-12-04T20:20:36,786 DEBUG [RS:0;c2ef38372881:40035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:20:36,788 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39095, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:20:36,789 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33473 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,40035,1733343636559 2024-12-04T20:20:36,789 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33473 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,40035,1733343636559 2024-12-04T20:20:36,790 DEBUG [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28 2024-12-04T20:20:36,790 DEBUG [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43357 2024-12-04T20:20:36,791 DEBUG [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:20:36,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:20:36,792 DEBUG [RS:0;c2ef38372881:40035 {}] zookeeper.ZKUtil(111): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,40035,1733343636559 2024-12-04T20:20:36,792 WARN [RS:0;c2ef38372881:40035 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:20:36,792 INFO [RS:0;c2ef38372881:40035 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:20:36,793 DEBUG [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559 2024-12-04T20:20:36,793 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,40035,1733343636559] 2024-12-04T20:20:36,796 INFO [RS:0;c2ef38372881:40035 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:20:36,798 INFO [RS:0;c2ef38372881:40035 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:20:36,798 INFO [RS:0;c2ef38372881:40035 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:20:36,798 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,798 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:20:36,799 INFO [RS:0;c2ef38372881:40035 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:20:36,799 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,799 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,799 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,799 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:20:36,800 DEBUG [RS:0;c2ef38372881:40035 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:20:36,801 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,801 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,801 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,801 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,801 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,801 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,40035,1733343636559-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:20:36,814 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:20:36,814 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,40035,1733343636559-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,815 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,815 INFO [RS:0;c2ef38372881:40035 {}] regionserver.Replication(171): c2ef38372881,40035,1733343636559 started 2024-12-04T20:20:36,826 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:36,827 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,40035,1733343636559, RpcServer on c2ef38372881/172.17.0.2:40035, sessionid=0x100a6e448800001 2024-12-04T20:20:36,827 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:20:36,827 DEBUG [RS:0;c2ef38372881:40035 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,40035,1733343636559 2024-12-04T20:20:36,827 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,40035,1733343636559' 2024-12-04T20:20:36,827 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:20:36,827 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:20:36,828 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:20:36,828 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:20:36,828 DEBUG [RS:0;c2ef38372881:40035 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,40035,1733343636559 2024-12-04T20:20:36,828 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,40035,1733343636559' 2024-12-04T20:20:36,828 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:20:36,829 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:20:36,829 DEBUG [RS:0;c2ef38372881:40035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:20:36,829 INFO [RS:0;c2ef38372881:40035 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:20:36,829 INFO [RS:0;c2ef38372881:40035 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:20:36,902 WARN [c2ef38372881:33473 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T20:20:36,933 INFO [RS:0;c2ef38372881:40035 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C40035%2C1733343636559, suffix=, logDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559, archiveDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/oldWALs, maxLogs=32 2024-12-04T20:20:36,936 INFO [RS:0;c2ef38372881:40035 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C40035%2C1733343636559.1733343636935 2024-12-04T20:20:36,945 INFO [RS:0;c2ef38372881:40035 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 2024-12-04T20:20:36,946 DEBUG [RS:0;c2ef38372881:40035 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45143:45143),(127.0.0.1/127.0.0.1:42877:42877)] 2024-12-04T20:20:37,153 DEBUG [c2ef38372881:33473 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T20:20:37,154 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,40035,1733343636559 2024-12-04T20:20:37,156 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,40035,1733343636559, state=OPENING 2024-12-04T20:20:37,158 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:20:37,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:37,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:20:37,161 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:20:37,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:20:37,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,40035,1733343636559}] 2024-12-04T20:20:37,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:20:37,317 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:20:37,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37541, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:20:37,326 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:20:37,326 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:20:37,327 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C40035%2C1733343636559.meta, suffix=.meta, logDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559, archiveDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/oldWALs, maxLogs=32 2024-12-04T20:20:37,328 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta 2024-12-04T20:20:37,333 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta 2024-12-04T20:20:37,334 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45143:45143),(127.0.0.1/127.0.0.1:42877:42877)] 2024-12-04T20:20:37,334 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:20:37,335 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:20:37,335 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:20:37,335 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:20:37,335 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:20:37,335 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:20:37,335 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:20:37,335 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:20:37,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:20:37,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:20:37,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:37,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:37,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:20:37,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:20:37,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:37,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:37,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:20:37,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:20:37,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:37,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:37,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:20:37,342 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:20:37,342 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:37,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:20:37,343 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:20:37,344 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740 2024-12-04T20:20:37,345 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740 2024-12-04T20:20:37,347 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:20:37,347 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:20:37,348 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:20:37,349 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:20:37,351 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=742472, jitterRate=-0.05589868128299713}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:20:37,351 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:20:37,351 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733343637335Writing region info on filesystem at 1733343637335Initializing all the Stores at 1733343637336 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343637336Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343637336Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343637336Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343637336Cleaning up temporary data from old regions at 1733343637347 (+11 ms)Running coprocessor post-open hooks at 1733343637351 (+4 ms)Region opened successfully at 1733343637351 2024-12-04T20:20:37,353 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733343637316 2024-12-04T20:20:37,357 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:20:37,357 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:20:37,357 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,40035,1733343636559 2024-12-04T20:20:37,358 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,40035,1733343636559, state=OPEN 2024-12-04T20:20:37,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:20:37,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:20:37,360 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,40035,1733343636559 2024-12-04T20:20:37,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:20:37,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:20:37,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:20:37,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,40035,1733343636559 in 199 msec 2024-12-04T20:20:37,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:20:37,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 613 msec 2024-12-04T20:20:37,367 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:20:37,367 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:20:37,368 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:20:37,368 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,40035,1733343636559, seqNum=-1] 2024-12-04T20:20:37,368 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:20:37,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53049, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:20:37,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 673 msec 2024-12-04T20:20:37,376 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733343637375, completionTime=-1 2024-12-04T20:20:37,376 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T20:20:37,376 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733343697378 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733343757378 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33473,1733343636517-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33473,1733343636517-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33473,1733343636517-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:33473, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:37,378 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:37,379 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:37,381 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:20:37,383 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.799sec 2024-12-04T20:20:37,383 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:20:37,383 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:20:37,383 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:20:37,383 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:20:37,383 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:20:37,383 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33473,1733343636517-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:20:37,383 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33473,1733343636517-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:20:37,386 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:20:37,386 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:20:37,386 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33473,1733343636517-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:20:37,387 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:37,392 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:37,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9f6f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:20:37,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,33473,-1 for getting cluster id 2024-12-04T20:20:37,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:20:37,474 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2725106b-318e-4384-b29c-c4c35fb6a122' 2024-12-04T20:20:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:20:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2725106b-318e-4384-b29c-c4c35fb6a122" 2024-12-04T20:20:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b3b172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:20:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,33473,-1] 2024-12-04T20:20:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:20:37,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:20:37,477 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:20:37,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db1f6e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:20:37,479 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:20:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,40035,1733343636559, seqNum=-1] 2024-12-04T20:20:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:20:37,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:20:37,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,33473,1733343636517 2024-12-04T20:20:37,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:20:37,488 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T20:20:37,488 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-12-04T20:20:37,489 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-12-04T20:20:37,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:20:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,33473,1733343636517 2024-12-04T20:20:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67930ae0 2024-12-04T20:20:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:20:37,492 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:20:37,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33473 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T20:20:37,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33473 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T20:20:37,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33473 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:20:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33473 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T20:20:37,497 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:20:37,497 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:37,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33473 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-12-04T20:20:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33473 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:20:37,499 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:20:37,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741835_1011 (size=395) 2024-12-04T20:20:37,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741835_1011 (size=395) 2024-12-04T20:20:37,510 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0f8fd95f5534dc77f6f8499e24885ea5, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28 2024-12-04T20:20:37,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741836_1012 (size=78) 2024-12-04T20:20:37,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46497 is added to blk_1073741836_1012 (size=78) 2024-12-04T20:20:37,919 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:20:37,920 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 0f8fd95f5534dc77f6f8499e24885ea5, disabling compactions & flushes 2024-12-04T20:20:37,920 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:20:37,920 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:20:37,920 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. after waiting 0 ms 2024-12-04T20:20:37,920 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:20:37,920 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:20:37,920 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0f8fd95f5534dc77f6f8499e24885ea5: Waiting for close lock at 1733343637919Disabling compacts and flushes for region at 1733343637919Disabling writes for close at 1733343637920 (+1 ms)Writing region close event to WAL at 1733343637920Closed at 1733343637920 2024-12-04T20:20:37,924 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:20:37,925 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733343637924"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733343637924"}]},"ts":"1733343637924"} 2024-12-04T20:20:37,931 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:20:37,934 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:20:37,934 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343637934"}]},"ts":"1733343637934"} 2024-12-04T20:20:37,937 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-12-04T20:20:37,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0f8fd95f5534dc77f6f8499e24885ea5, ASSIGN}] 2024-12-04T20:20:37,939 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0f8fd95f5534dc77f6f8499e24885ea5, ASSIGN 2024-12-04T20:20:37,940 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0f8fd95f5534dc77f6f8499e24885ea5, ASSIGN; state=OFFLINE, location=c2ef38372881,40035,1733343636559; forceNewPlan=false, retain=false 2024-12-04T20:20:38,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0f8fd95f5534dc77f6f8499e24885ea5, regionState=OPENING, regionLocation=c2ef38372881,40035,1733343636559 2024-12-04T20:20:38,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0f8fd95f5534dc77f6f8499e24885ea5, ASSIGN because future has completed 2024-12-04T20:20:38,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f8fd95f5534dc77f6f8499e24885ea5, server=c2ef38372881,40035,1733343636559}] 2024-12-04T20:20:38,250 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:20:38,250 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f8fd95f5534dc77f6f8499e24885ea5, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:20:38,250 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,250 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:20:38,250 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,250 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,252 INFO [StoreOpener-0f8fd95f5534dc77f6f8499e24885ea5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,253 INFO [StoreOpener-0f8fd95f5534dc77f6f8499e24885ea5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f8fd95f5534dc77f6f8499e24885ea5 columnFamilyName info 2024-12-04T20:20:38,253 DEBUG [StoreOpener-0f8fd95f5534dc77f6f8499e24885ea5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:20:38,254 INFO [StoreOpener-0f8fd95f5534dc77f6f8499e24885ea5-1 {}] regionserver.HStore(327): Store=0f8fd95f5534dc77f6f8499e24885ea5/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:20:38,254 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,254 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,255 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,255 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,255 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,257 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,258 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:20:38,259 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0f8fd95f5534dc77f6f8499e24885ea5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881737, jitterRate=0.1211870014667511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:20:38,259 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:20:38,260 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0f8fd95f5534dc77f6f8499e24885ea5: Running coprocessor pre-open hook at 1733343638250Writing region info on filesystem at 1733343638250Initializing all the Stores at 1733343638251 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343638251Cleaning up temporary data from old regions at 1733343638255 (+4 ms)Running coprocessor post-open hooks at 1733343638259 (+4 ms)Region opened successfully at 1733343638260 (+1 ms) 2024-12-04T20:20:38,261 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5., pid=6, masterSystemTime=1733343638246 2024-12-04T20:20:38,263 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:20:38,263 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:20:38,264 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0f8fd95f5534dc77f6f8499e24885ea5, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,40035,1733343636559 2024-12-04T20:20:38,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f8fd95f5534dc77f6f8499e24885ea5, server=c2ef38372881,40035,1733343636559 because future has completed 2024-12-04T20:20:38,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T20:20:38,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0f8fd95f5534dc77f6f8499e24885ea5, server=c2ef38372881,40035,1733343636559 in 175 msec 2024-12-04T20:20:38,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T20:20:38,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0f8fd95f5534dc77f6f8499e24885ea5, ASSIGN in 335 msec 2024-12-04T20:20:38,276 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:20:38,276 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343638276"}]},"ts":"1733343638276"} 2024-12-04T20:20:38,279 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-12-04T20:20:38,280 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:20:38,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 786 msec 2024-12-04T20:20:38,387 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:38,393 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:39,388 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:39,393 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:40,389 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:40,394 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:41,390 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:41,395 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:42,391 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:42,396 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:42,841 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T20:20:42,858 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,859 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,859 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,859 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,859 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,860 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,862 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,862 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,862 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,864 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:20:42,868 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:20:42,868 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T20:20:42,868 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T20:20:42,868 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-12-04T20:20:42,868 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:20:42,868 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T20:20:42,869 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T20:20:42,869 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-12-04T20:20:43,392 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:43,397 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:44,393 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:44,398 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:45,394 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:45,399 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:46,395 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:46,399 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:47,396 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:47,401 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33473 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:20:47,583 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-12-04T20:20:47,583 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-12-04T20:20:47,590 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T20:20:47,590 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:20:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5., hostname=c2ef38372881,40035,1733343636559, seqNum=2] 2024-12-04T20:20:48,398 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:48,402 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:49,399 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:49,403 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:49,597 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 2024-12-04T20:20:49,598 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:49,598 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:46497,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:49,598 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:46497,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:49,599 WARN [DataStreamer for file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 block BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46497,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK], DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46497,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]) is bad. 2024-12-04T20:20:49,599 WARN [DataStreamer for file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta block BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK], DatanodeInfoWithStorage[127.0.0.1:46497,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46497,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]) is bad. 2024-12-04T20:20:49,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1794681859_22 at /127.0.0.1:48868 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34201:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48868 dst: /127.0.0.1:34201 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1794681859_22 at /127.0.0.1:54462 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46497:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54462 dst: /127.0.0.1:46497 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,600 WARN [PacketResponder: BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46497] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,600 WARN [DataStreamer for file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 block BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK], DatanodeInfoWithStorage[127.0.0.1:46497,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46497,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]) is bad. 2024-12-04T20:20:49,601 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:54480 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46497:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54480 dst: /127.0.0.1:46497 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:48904 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34201:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48904 dst: /127.0.0.1:34201 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,602 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:48910 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34201:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48910 dst: /127.0.0.1:34201 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,602 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:54486 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46497:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54486 dst: /127.0.0.1:46497 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,603 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9982f0a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:49,604 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70df7796{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:49,604 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:49,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4360f0f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:49,605 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28441b3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:49,607 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:49,607 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:49,607 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1711332990-172.17.0.2-1733343635930 (Datanode Uuid 9f6ab646-fabd-46cc-b732-79c4649a8ae1) service to localhost/127.0.0.1:43357 2024-12-04T20:20:49,607 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:49,608 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data3/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:49,608 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data4/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:49,608 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:49,617 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:49,621 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:49,622 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:49,622 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:49,622 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:20:49,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@431e378c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:49,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68e23717{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:49,715 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21595673{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/java.io.tmpdir/jetty-localhost-38623-hadoop-hdfs-3_4_1-tests_jar-_-any-8655249310683347791/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:49,715 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c50d76a{HTTP/1.1, (http/1.1)}{localhost:38623} 2024-12-04T20:20:49,715 INFO [Time-limited test {}] server.Server(415): Started @161555ms 2024-12-04T20:20:49,716 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:49,733 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:49,733 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:49,733 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:49,734 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1794681859_22 at /127.0.0.1:50606 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34201:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50606 dst: /127.0.0.1:34201 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,734 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:50630 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34201:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50630 dst: /127.0.0.1:34201 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,734 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:50622 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34201:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50622 dst: /127.0.0.1:34201 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:49,737 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6e0e18a9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:49,737 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@a5d6d97{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:49,737 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:49,737 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74ea1d44{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:49,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@66c0323e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:49,739 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:49,739 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:49,739 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:49,739 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1711332990-172.17.0.2-1733343635930 (Datanode Uuid 81123f1c-02df-4806-bcc6-1ecb87f7a5bf) service to localhost/127.0.0.1:43357 2024-12-04T20:20:49,739 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data1/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:49,739 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data2/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:49,740 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:49,769 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:49,771 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:49,773 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:49,773 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:49,773 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:20:49,774 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62f6e774{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:49,775 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31267d1e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:49,807 WARN [Thread-1353 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:49,809 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb5912427fcf5c77a with lease ID 0xafa9e5de11572c5: from storage DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3 node DatanodeRegistration(127.0.0.1:34385, datanodeUuid=9f6ab646-fabd-46cc-b732-79c4649a8ae1, infoPort=40495, infoSecurePort=0, ipcPort=44057, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:49,810 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb5912427fcf5c77a with lease ID 0xafa9e5de11572c5: from storage DS-5c47abdd-12e0-479a-9821-69d4b9e569a6 node DatanodeRegistration(127.0.0.1:34385, datanodeUuid=9f6ab646-fabd-46cc-b732-79c4649a8ae1, infoPort=40495, infoSecurePort=0, ipcPort=44057, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:49,866 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70770496{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/java.io.tmpdir/jetty-localhost-44003-hadoop-hdfs-3_4_1-tests_jar-_-any-1688738829392977796/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:49,867 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@542bdd12{HTTP/1.1, (http/1.1)}{localhost:44003} 2024-12-04T20:20:49,867 INFO [Time-limited test {}] server.Server(415): Started @161707ms 2024-12-04T20:20:49,868 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:49,932 WARN [Thread-1384 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:49,934 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3798311cb82a5901 with lease ID 0xafa9e5de11572c6: from storage DS-1e373622-0db0-4fff-8d45-0a62e726ebcf node DatanodeRegistration(127.0.0.1:41531, datanodeUuid=81123f1c-02df-4806-bcc6-1ecb87f7a5bf, infoPort=38341, infoSecurePort=0, ipcPort=39691, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:49,934 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3798311cb82a5901 with lease ID 0xafa9e5de11572c6: from storage DS-947c3f6e-224f-479e-bca0-5a3efadfdd8d node DatanodeRegistration(127.0.0.1:41531, datanodeUuid=81123f1c-02df-4806-bcc6-1ecb87f7a5bf, infoPort=38341, infoSecurePort=0, ipcPort=39691, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:50,400 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:50,404 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:50,884 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-12-04T20:20:50,887 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-12-04T20:20:50,888 ERROR [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28-prefix:c2ef38372881,40035,1733343636559 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:50,888 WARN [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28-prefix:c2ef38372881,40035,1733343636559 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:50,888 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C40035%2C1733343636559:(num 1733343636935) roll requested 2024-12-04T20:20:50,889 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C40035%2C1733343636559.1733343650889 2024-12-04T20:20:50,899 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 newFile=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 2024-12-04T20:20:50,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:50,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:50,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:50,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:50,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:50,900 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 2024-12-04T20:20:50,900 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:50,900 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:50,900 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 2024-12-04T20:20:50,901 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40495:40495),(127.0.0.1/127.0.0.1:38341:38341)] 2024-12-04T20:20:50,901 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 is not closed yet, will try archiving it next time 2024-12-04T20:20:50,901 WARN [IPC Server handler 3 on default port 43357 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1015 2024-12-04T20:20:50,901 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 after 1ms 2024-12-04T20:20:51,401 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:51,405 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:51,810 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1015: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T20:20:52,402 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:52,405 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:52,904 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-12-04T20:20:53,402 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:53,406 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:54,403 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:54,406 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:54,903 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 after 4003ms 2024-12-04T20:20:54,909 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016 java.io.IOException: Bad response ERROR for BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:41531,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:54,909 WARN [DataStreamer for file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 block BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34385,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK], DatanodeInfoWithStorage[127.0.0.1:41531,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41531,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]) is bad. 2024-12-04T20:20:54,910 WARN [PacketResponder: BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41531] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:54,911 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:50552 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:34385:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50552 dst: /127.0.0.1:34385 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:54,911 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:58024 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:41531:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58024 dst: /127.0.0.1:41531 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:54,913 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70770496{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:54,914 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@542bdd12{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:54,914 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:54,914 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31267d1e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:54,914 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62f6e774{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:54,916 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:54,916 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:54,916 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:54,916 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1711332990-172.17.0.2-1733343635930 (Datanode Uuid 81123f1c-02df-4806-bcc6-1ecb87f7a5bf) service to localhost/127.0.0.1:43357 2024-12-04T20:20:54,917 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data1/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:54,918 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data2/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:54,918 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:54,926 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:54,928 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:54,929 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:54,929 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:54,929 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:20:54,930 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32c717fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:54,931 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20432799{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:55,020 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5aca21ee{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/java.io.tmpdir/jetty-localhost-40193-hadoop-hdfs-3_4_1-tests_jar-_-any-8106551224390596532/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:55,020 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@570cb725{HTTP/1.1, (http/1.1)}{localhost:40193} 2024-12-04T20:20:55,020 INFO [Time-limited test {}] server.Server(415): Started @166860ms 2024-12-04T20:20:55,021 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:55,043 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:55,043 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1799411466_22 at /127.0.0.1:50568 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:34385:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50568 dst: /127.0.0.1:34385 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:55,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21595673{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:55,052 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c50d76a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:20:55,052 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:20:55,052 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68e23717{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:20:55,052 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@431e378c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,STOPPED} 2024-12-04T20:20:55,053 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:20:55,053 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:20:55,053 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1711332990-172.17.0.2-1733343635930 (Datanode Uuid 9f6ab646-fabd-46cc-b732-79c4649a8ae1) service to localhost/127.0.0.1:43357 2024-12-04T20:20:55,053 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:20:55,054 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data3/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:55,054 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data4/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:20:55,054 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:20:55,069 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:20:55,073 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:20:55,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:20:55,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:20:55,078 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:20:55,078 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ebbad67{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:20:55,078 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1decdda3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:20:55,118 WARN [Thread-1427 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaadc2608985313dc with lease ID 0xafa9e5de11572c7: from storage DS-1e373622-0db0-4fff-8d45-0a62e726ebcf node DatanodeRegistration(127.0.0.1:41775, datanodeUuid=81123f1c-02df-4806-bcc6-1ecb87f7a5bf, infoPort=38525, infoSecurePort=0, ipcPort=37527, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaadc2608985313dc with lease ID 0xafa9e5de11572c7: from storage DS-947c3f6e-224f-479e-bca0-5a3efadfdd8d node DatanodeRegistration(127.0.0.1:41775, datanodeUuid=81123f1c-02df-4806-bcc6-1ecb87f7a5bf, infoPort=38525, infoSecurePort=0, ipcPort=37527, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:55,171 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@73f17350{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/java.io.tmpdir/jetty-localhost-35797-hadoop-hdfs-3_4_1-tests_jar-_-any-3075020545980045549/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:20:55,171 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7da73466{HTTP/1.1, (http/1.1)}{localhost:35797} 2024-12-04T20:20:55,172 INFO [Time-limited test {}] server.Server(415): Started @167012ms 2024-12-04T20:20:55,173 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:20:55,235 WARN [Thread-1458 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:20:55,238 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd608d4822c37d8a9 with lease ID 0xafa9e5de11572c8: from storage DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3 node DatanodeRegistration(127.0.0.1:34695, datanodeUuid=9f6ab646-fabd-46cc-b732-79c4649a8ae1, infoPort=46617, infoSecurePort=0, ipcPort=35087, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:55,238 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd608d4822c37d8a9 with lease ID 0xafa9e5de11572c8: from storage DS-5c47abdd-12e0-479a-9821-69d4b9e569a6 node DatanodeRegistration(127.0.0.1:34695, datanodeUuid=9f6ab646-fabd-46cc-b732-79c4649a8ae1, infoPort=46617, infoSecurePort=0, ipcPort=35087, storageInfo=lv=-57;cid=testClusterID;nsid=150223945;c=1733343635930), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:20:55,404 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:55,407 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:56,194 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-12-04T20:20:56,196 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-12-04T20:20:56,197 ERROR [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28-prefix:c2ef38372881,40035,1733343636559 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34385,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:56,197 WARN [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28-prefix:c2ef38372881,40035,1733343636559 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34385,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:56,197 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C40035%2C1733343636559:(num 1733343650889) roll requested 2024-12-04T20:20:56,198 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C40035%2C1733343636559.1733343656198 2024-12-04T20:20:56,204 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 newFile=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343656198 2024-12-04T20:20:56,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:56,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:56,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:56,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:56,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:56,204 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343656198 2024-12-04T20:20:56,205 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34385,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:56,205 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34385,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:56,205 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 2024-12-04T20:20:56,205 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38525:38525),(127.0.0.1/127.0.0.1:46617:46617)] 2024-12-04T20:20:56,205 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 is not closed yet, will try archiving it next time 2024-12-04T20:20:56,205 WARN [IPC Server handler 1 on default port 43357 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-12-04T20:20:56,206 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 after 1ms 2024-12-04T20:20:56,404 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:56,408 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:57,119 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T20:20:57,405 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:57,408 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:58,207 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:20:58,214 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343656198 newFile=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:20:58,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:58,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:58,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:58,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:58,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:20:58,215 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343656198 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:20:58,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741838_1019 (size=1264) 2024-12-04T20:20:58,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741838_1019 (size=1264) 2024-12-04T20:20:58,217 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 is not closed yet, will try archiving it next time 2024-12-04T20:20:58,229 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38525:38525),(127.0.0.1/127.0.0.1:46617:46617)] 2024-12-04T20:20:58,229 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 is not closed yet, will try archiving it next time 2024-12-04T20:20:58,230 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 2024-12-04T20:20:58,230 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 2024-12-04T20:20:58,230 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 after 0ms 2024-12-04T20:20:58,230 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 2024-12-04T20:20:58,241 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733343638260/Put/vlen=218/seqid=0] 2024-12-04T20:20:58,242 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733343647595/Put/vlen=1045/seqid=0] 2024-12-04T20:20:58,242 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343636935 2024-12-04T20:20:58,242 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 2024-12-04T20:20:58,242 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 2024-12-04T20:20:58,243 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 after 1ms 2024-12-04T20:20:58,243 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 2024-12-04T20:20:58,248 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733343650888/Put/vlen=1045/seqid=0] 2024-12-04T20:20:58,248 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733343652905/Put/vlen=1045/seqid=0] 2024-12-04T20:20:58,248 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 2024-12-04T20:20:58,248 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343656198 2024-12-04T20:20:58,248 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343656198 2024-12-04T20:20:58,249 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343656198 after 1ms 2024-12-04T20:20:58,249 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343656198 2024-12-04T20:20:58,253 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733343656197/Put/vlen=1045/seqid=0] 2024-12-04T20:20:58,253 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:20:58,253 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:20:58,254 WARN [IPC Server handler 2 on default port 43357 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-12-04T20:20:58,254 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 after 1ms 2024-12-04T20:20:58,406 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:58,409 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:59,242 WARN [ResponseProcessor for block BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:59,242 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1794681859_22 at /127.0.0.1:59968 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59968 dst: /127.0.0.1:41775 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41775 remote=/127.0.0.1:59968]. Total timeout mills is 60000, 58971 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:59,243 WARN [DataStreamer for file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 block BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41775,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK], DatanodeInfoWithStorage[127.0.0.1:34695,DS-f6aef71f-7fd0-458b-8e2c-2e3eb75a9df3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41775,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]) is bad. 2024-12-04T20:20:59,243 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1794681859_22 at /127.0.0.1:49170 [Receiving block BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:34695:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49170 dst: /127.0.0.1:34695 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:20:59,244 WARN [DataStreamer for file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 block BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:20:59,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741839_1022 (size=85) 2024-12-04T20:20:59,407 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:20:59,409 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:00,207 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343650889 after 4002ms 2024-12-04T20:21:00,407 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:00,409 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:01,408 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:01,410 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:02,255 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 after 4002ms 2024-12-04T20:21:02,255 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:21:02,259 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:21:02,259 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-12-04T20:21:02,260 ERROR [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28-prefix:c2ef38372881,40035,1733343636559.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:02,260 WARN [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28-prefix:c2ef38372881,40035,1733343636559.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:02,260 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C40035%2C1733343636559.meta:.meta(num 1733343637328) roll requested 2024-12-04T20:21:02,261 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C40035%2C1733343636559.meta.1733343662260.meta 2024-12-04T20:21:02,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,267 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343662260.meta 2024-12-04T20:21:02,267 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:02,267 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:02,267 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta 2024-12-04T20:21:02,268 WARN [IPC Server handler 3 on default port 43357 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1014 2024-12-04T20:21:02,268 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46617:46617),(127.0.0.1/127.0.0.1:38525:38525)] 2024-12-04T20:21:02,268 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta is not closed yet, will try archiving it next time 2024-12-04T20:21:02,268 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta after 1ms 2024-12-04T20:21:02,282 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/info/c16cb328c1104ddf80400b7d8983b3cf is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5./info:regioninfo/1733343638264/Put/seqid=0 2024-12-04T20:21:02,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741841_1025 (size=7125) 2024-12-04T20:21:02,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741841_1025 (size=7125) 2024-12-04T20:21:02,287 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/info/c16cb328c1104ddf80400b7d8983b3cf 2024-12-04T20:21:02,305 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/ns/d2048ad48466427f925e3afe1022371c is 43, key is default/ns:d/1733343637370/Put/seqid=0 2024-12-04T20:21:02,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741842_1026 (size=5153) 2024-12-04T20:21:02,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741842_1026 (size=5153) 2024-12-04T20:21:02,310 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/ns/d2048ad48466427f925e3afe1022371c 2024-12-04T20:21:02,327 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/table/bc23266c217949d3990f8db77ecce780 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733343638276/Put/seqid=0 2024-12-04T20:21:02,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741843_1027 (size=5438) 2024-12-04T20:21:02,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741843_1027 (size=5438) 2024-12-04T20:21:02,332 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/table/bc23266c217949d3990f8db77ecce780 2024-12-04T20:21:02,338 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/info/c16cb328c1104ddf80400b7d8983b3cf as hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/info/c16cb328c1104ddf80400b7d8983b3cf 2024-12-04T20:21:02,343 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/info/c16cb328c1104ddf80400b7d8983b3cf, entries=10, sequenceid=11, filesize=7.0 K 2024-12-04T20:21:02,344 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/ns/d2048ad48466427f925e3afe1022371c as hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/ns/d2048ad48466427f925e3afe1022371c 2024-12-04T20:21:02,349 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/ns/d2048ad48466427f925e3afe1022371c, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T20:21:02,350 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/.tmp/table/bc23266c217949d3990f8db77ecce780 as hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/table/bc23266c217949d3990f8db77ecce780 2024-12-04T20:21:02,356 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/table/bc23266c217949d3990f8db77ecce780, entries=2, sequenceid=11, filesize=5.3 K 2024-12-04T20:21:02,358 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 99ms, sequenceid=11, compaction requested=false 2024-12-04T20:21:02,358 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-04T20:21:02,358 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0f8fd95f5534dc77f6f8499e24885ea5 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-12-04T20:21:02,358 ERROR [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28-prefix:c2ef38372881,40035,1733343636559 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:02,359 WARN [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28-prefix:c2ef38372881,40035,1733343636559 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:02,359 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C40035%2C1733343636559:(num 1733343658207) roll requested 2024-12-04T20:21:02,359 INFO [regionserver/c2ef38372881:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C40035%2C1733343636559.1733343662359 2024-12-04T20:21:02,367 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 newFile=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343662359 2024-12-04T20:21:02,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,367 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343662359 2024-12-04T20:21:02,367 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:02,368 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46617:46617),(127.0.0.1/127.0.0.1:38525:38525)] 2024-12-04T20:21:02,368 DEBUG [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 is not closed yet, will try archiving it next time 2024-12-04T20:21:02,368 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1711332990-172.17.0.2-1733343635930:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:02,368 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:21:02,368 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 after 0ms 2024-12-04T20:21:02,369 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.1733343658207 to hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/oldWALs/c2ef38372881%2C40035%2C1733343636559.1733343658207 2024-12-04T20:21:02,381 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5/.tmp/info/c42afdab3d834389976094a0dcf3fa30 is 1080, key is row1002/info:/1733343647595/Put/seqid=0 2024-12-04T20:21:02,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741845_1029 (size=9270) 2024-12-04T20:21:02,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741845_1029 (size=9270) 2024-12-04T20:21:02,387 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5/.tmp/info/c42afdab3d834389976094a0dcf3fa30 2024-12-04T20:21:02,394 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5/.tmp/info/c42afdab3d834389976094a0dcf3fa30 as hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5/info/c42afdab3d834389976094a0dcf3fa30 2024-12-04T20:21:02,401 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5/info/c42afdab3d834389976094a0dcf3fa30, entries=4, sequenceid=8, filesize=9.1 K 2024-12-04T20:21:02,402 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 0f8fd95f5534dc77f6f8499e24885ea5 in 44ms, sequenceid=8, compaction requested=false 2024-12-04T20:21:02,402 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0f8fd95f5534dc77f6f8499e24885ea5: 2024-12-04T20:21:02,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:21:02,408 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:21:02,408 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:21:02,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:21:02,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:21:02,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:21:02,409 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:21:02,408 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:02,409 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=679989252, stopped=false 2024-12-04T20:21:02,409 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,33473,1733343636517 2024-12-04T20:21:02,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:21:02,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:02,410 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:02,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:21:02,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:02,411 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:21:02,411 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:21:02,411 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:21:02,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:21:02,411 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,40035,1733343636559' ***** 2024-12-04T20:21:02,411 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:21:02,412 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:21:02,412 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:21:02,412 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:21:02,412 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:21:02,412 INFO [RS:0;c2ef38372881:40035 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:21:02,412 INFO [RS:0;c2ef38372881:40035 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:21:02,412 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(3091): Received CLOSE for 0f8fd95f5534dc77f6f8499e24885ea5 2024-12-04T20:21:02,413 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,40035,1733343636559 2024-12-04T20:21:02,413 INFO [RS:0;c2ef38372881:40035 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:21:02,413 INFO [RS:0;c2ef38372881:40035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:40035. 2024-12-04T20:21:02,413 DEBUG [RS:0;c2ef38372881:40035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:21:02,413 DEBUG [RS:0;c2ef38372881:40035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:21:02,413 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0f8fd95f5534dc77f6f8499e24885ea5, disabling compactions & flushes 2024-12-04T20:21:02,413 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:21:02,413 INFO [RS:0;c2ef38372881:40035 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:21:02,413 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:21:02,413 INFO [RS:0;c2ef38372881:40035 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:21:02,413 INFO [RS:0;c2ef38372881:40035 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:21:02,413 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. after waiting 0 ms 2024-12-04T20:21:02,413 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:21:02,413 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:21:02,414 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T20:21:02,414 DEBUG [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 0f8fd95f5534dc77f6f8499e24885ea5=TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5.} 2024-12-04T20:21:02,414 DEBUG [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1351): Waiting on 0f8fd95f5534dc77f6f8499e24885ea5, 1588230740 2024-12-04T20:21:02,414 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:21:02,414 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:21:02,414 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:21:02,414 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:21:02,415 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:21:02,418 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/default/TestLogRolling-testLogRollOnPipelineRestart/0f8fd95f5534dc77f6f8499e24885ea5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T20:21:02,419 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:21:02,419 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0f8fd95f5534dc77f6f8499e24885ea5: Waiting for close lock at 1733343662413Running coprocessor pre-close hooks at 1733343662413Disabling compacts and flushes for region at 1733343662413Disabling writes for close at 1733343662413Writing region close event to WAL at 1733343662414 (+1 ms)Running coprocessor post-close hooks at 1733343662419 (+5 ms)Closed at 1733343662419 2024-12-04T20:21:02,419 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733343637493.0f8fd95f5534dc77f6f8499e24885ea5. 2024-12-04T20:21:02,421 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T20:21:02,422 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:21:02,422 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:21:02,422 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343662414Running coprocessor pre-close hooks at 1733343662414Disabling compacts and flushes for region at 1733343662414Disabling writes for close at 1733343662414Writing region close event to WAL at 1733343662418 (+4 ms)Running coprocessor post-close hooks at 1733343662422 (+4 ms)Closed at 1733343662422 2024-12-04T20:21:02,422 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:21:02,615 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,40035,1733343636559; all regions closed. 2024-12-04T20:21:02,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:02,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:21:02,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:21:02,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741840_1023 (size=825) 2024-12-04T20:21:02,622 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T20:21:02,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741840_1023 (size=825) 2024-12-04T20:21:02,801 INFO [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T20:21:02,802 INFO [regionserver/c2ef38372881:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T20:21:02,803 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:21:03,409 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:03,411 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:04,410 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:04,411 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:05,410 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:05,412 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:06,239 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T20:21:06,269 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta after 4002ms 2024-12-04T20:21:06,269 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/WALs/c2ef38372881,40035,1733343636559/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta to hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/oldWALs/c2ef38372881%2C40035%2C1733343636559.meta.1733343637328.meta 2024-12-04T20:21:06,272 DEBUG [RS:0;c2ef38372881:40035 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/oldWALs 2024-12-04T20:21:06,272 INFO [RS:0;c2ef38372881:40035 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C40035%2C1733343636559.meta:.meta(num 1733343662260) 2024-12-04T20:21:06,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741844_1028 (size=1162) 2024-12-04T20:21:06,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741844_1028 (size=1162) 2024-12-04T20:21:06,279 DEBUG [RS:0;c2ef38372881:40035 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/oldWALs 2024-12-04T20:21:06,279 INFO [RS:0;c2ef38372881:40035 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C40035%2C1733343636559:(num 1733343662359) 2024-12-04T20:21:06,279 DEBUG [RS:0;c2ef38372881:40035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:21:06,279 INFO [RS:0;c2ef38372881:40035 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:21:06,279 INFO [RS:0;c2ef38372881:40035 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:21:06,279 INFO [RS:0;c2ef38372881:40035 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T20:21:06,279 INFO [RS:0;c2ef38372881:40035 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:21:06,279 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:21:06,279 INFO [RS:0;c2ef38372881:40035 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40035 2024-12-04T20:21:06,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,40035,1733343636559 2024-12-04T20:21:06,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:21:06,280 INFO [RS:0;c2ef38372881:40035 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:21:06,281 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,40035,1733343636559] 2024-12-04T20:21:06,282 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,40035,1733343636559 already deleted, retry=false 2024-12-04T20:21:06,282 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,40035,1733343636559 expired; onlineServers=0 2024-12-04T20:21:06,282 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,33473,1733343636517' ***** 2024-12-04T20:21:06,282 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:21:06,282 INFO [M:0;c2ef38372881:33473 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:21:06,282 INFO [M:0;c2ef38372881:33473 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:21:06,282 DEBUG [M:0;c2ef38372881:33473 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:21:06,282 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:21:06,282 DEBUG [M:0;c2ef38372881:33473 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:21:06,282 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343636706 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343636706,5,FailOnTimeoutGroup] 2024-12-04T20:21:06,282 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343636706 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343636706,5,FailOnTimeoutGroup] 2024-12-04T20:21:06,283 INFO [M:0;c2ef38372881:33473 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:21:06,283 INFO [M:0;c2ef38372881:33473 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:21:06,283 DEBUG [M:0;c2ef38372881:33473 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:21:06,283 INFO [M:0;c2ef38372881:33473 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:21:06,283 INFO [M:0;c2ef38372881:33473 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:21:06,283 INFO [M:0;c2ef38372881:33473 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:21:06,283 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:21:06,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:21:06,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:06,284 DEBUG [M:0;c2ef38372881:33473 {}] zookeeper.ZKUtil(347): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:21:06,284 WARN [M:0;c2ef38372881:33473 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:21:06,284 INFO [M:0;c2ef38372881:33473 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/.lastflushedseqids 2024-12-04T20:21:06,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741846_1030 (size=139) 2024-12-04T20:21:06,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741846_1030 (size=139) 2024-12-04T20:21:06,290 INFO [M:0;c2ef38372881:33473 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:21:06,290 INFO [M:0;c2ef38372881:33473 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:21:06,290 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:21:06,290 INFO [M:0;c2ef38372881:33473 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:21:06,290 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:21:06,290 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:21:06,290 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:21:06,290 INFO [M:0;c2ef38372881:33473 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB 2024-12-04T20:21:06,291 ERROR [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData-prefix:c2ef38372881,33473,1733343636517 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:06,291 WARN [FSHLog-0-hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData-prefix:c2ef38372881,33473,1733343636517 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:06,291 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog c2ef38372881%2C33473%2C1733343636517:(num 1733343636639) roll requested 2024-12-04T20:21:06,291 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33473%2C1733343636517.1733343666291 2024-12-04T20:21:06,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,296 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343666291 2024-12-04T20:21:06,297 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:06,297 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34201,DS-1e373622-0db0-4fff-8d45-0a62e726ebcf,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T20:21:06,297 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 2024-12-04T20:21:06,297 WARN [IPC Server handler 0 on default port 43357 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013 2024-12-04T20:21:06,297 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 after 0ms 2024-12-04T20:21:06,300 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38525:38525),(127.0.0.1/127.0.0.1:46617:46617)] 2024-12-04T20:21:06,300 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 is not closed yet, will try archiving it next time 2024-12-04T20:21:06,314 DEBUG [M:0;c2ef38372881:33473 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e840e794d1ad4b318cf6d3868c7e79a9 is 82, key is hbase:meta,,1/info:regioninfo/1733343637357/Put/seqid=0 2024-12-04T20:21:06,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741848_1033 (size=5672) 2024-12-04T20:21:06,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741848_1033 (size=5672) 2024-12-04T20:21:06,319 INFO [M:0;c2ef38372881:33473 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e840e794d1ad4b318cf6d3868c7e79a9 2024-12-04T20:21:06,338 DEBUG [M:0;c2ef38372881:33473 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e98d4226f5e845bc8b22f99444aa4b76 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733343638281/Put/seqid=0 2024-12-04T20:21:06,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741849_1034 (size=6118) 2024-12-04T20:21:06,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741849_1034 (size=6118) 2024-12-04T20:21:06,344 INFO [M:0;c2ef38372881:33473 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e98d4226f5e845bc8b22f99444aa4b76 2024-12-04T20:21:06,362 DEBUG [M:0;c2ef38372881:33473 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b947733ed89743e182cda4da4589a198 is 69, key is c2ef38372881,40035,1733343636559/rs:state/1733343636789/Put/seqid=0 2024-12-04T20:21:06,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741850_1035 (size=5156) 2024-12-04T20:21:06,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741850_1035 (size=5156) 2024-12-04T20:21:06,368 INFO [M:0;c2ef38372881:33473 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b947733ed89743e182cda4da4589a198 2024-12-04T20:21:06,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:21:06,381 INFO [RS:0;c2ef38372881:40035 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:21:06,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40035-0x100a6e448800001, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:21:06,381 INFO [RS:0;c2ef38372881:40035 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,40035,1733343636559; zookeeper connection closed. 2024-12-04T20:21:06,382 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@68bf899f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@68bf899f 2024-12-04T20:21:06,382 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T20:21:06,386 DEBUG [M:0;c2ef38372881:33473 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/45f7eafb20a14c688582f68facab7a3e is 52, key is load_balancer_on/state:d/1733343637487/Put/seqid=0 2024-12-04T20:21:06,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741851_1036 (size=5056) 2024-12-04T20:21:06,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741851_1036 (size=5056) 2024-12-04T20:21:06,391 INFO [M:0;c2ef38372881:33473 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/45f7eafb20a14c688582f68facab7a3e 2024-12-04T20:21:06,397 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e840e794d1ad4b318cf6d3868c7e79a9 as hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e840e794d1ad4b318cf6d3868c7e79a9 2024-12-04T20:21:06,403 INFO [M:0;c2ef38372881:33473 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e840e794d1ad4b318cf6d3868c7e79a9, entries=8, sequenceid=56, filesize=5.5 K 2024-12-04T20:21:06,404 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e98d4226f5e845bc8b22f99444aa4b76 as hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e98d4226f5e845bc8b22f99444aa4b76 2024-12-04T20:21:06,408 INFO [M:0;c2ef38372881:33473 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e98d4226f5e845bc8b22f99444aa4b76, entries=6, sequenceid=56, filesize=6.0 K 2024-12-04T20:21:06,409 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b947733ed89743e182cda4da4589a198 as hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b947733ed89743e182cda4da4589a198 2024-12-04T20:21:06,411 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:06,412 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:06,414 INFO [M:0;c2ef38372881:33473 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b947733ed89743e182cda4da4589a198, entries=1, sequenceid=56, filesize=5.0 K 2024-12-04T20:21:06,415 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/45f7eafb20a14c688582f68facab7a3e as hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/45f7eafb20a14c688582f68facab7a3e 2024-12-04T20:21:06,419 INFO [M:0;c2ef38372881:33473 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/45f7eafb20a14c688582f68facab7a3e, entries=1, sequenceid=56, filesize=4.9 K 2024-12-04T20:21:06,420 INFO [M:0;c2ef38372881:33473 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 130ms, sequenceid=56, compaction requested=false 2024-12-04T20:21:06,421 INFO [M:0;c2ef38372881:33473 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:21:06,421 DEBUG [M:0;c2ef38372881:33473 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343666290Disabling compacts and flushes for region at 1733343666290Disabling writes for close at 1733343666290Obtaining lock to block concurrent updates at 1733343666290Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733343666290Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1733343666291 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733343666301 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733343666301Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733343666314 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733343666314Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733343666325 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733343666338 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733343666338Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733343666349 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733343666362 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733343666362Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733343666372 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733343666386 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733343666386Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cf6a457: reopening flushed file at 1733343666396 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d67b709: reopening flushed file at 1733343666403 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fd7ff49: reopening flushed file at 1733343666409 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79161652: reopening flushed file at 1733343666414 (+5 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 130ms, sequenceid=56, compaction requested=false at 1733343666420 (+6 ms)Writing region close event to WAL at 1733343666421 (+1 ms)Closed at 1733343666421 2024-12-04T20:21:06,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:06,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34695 is added to blk_1073741847_1031 (size=757) 2024-12-04T20:21:06,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41775 is added to blk_1073741847_1031 (size=757) 2024-12-04T20:21:06,494 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:21:07,412 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:07,413 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:07,420 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,421 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,439 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,439 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,439 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,440 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,440 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,440 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,442 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,442 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,442 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,444 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,447 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,447 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,950 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T20:21:07,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,953 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,969 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,969 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,969 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,969 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,969 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,970 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,972 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,972 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,972 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:07,974 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:08,413 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:08,414 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:09,240 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T20:21:09,414 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:09,415 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:10,299 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 after 4001ms 2024-12-04T20:21:10,299 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/WALs/c2ef38372881,33473,1733343636517/c2ef38372881%2C33473%2C1733343636517.1733343636639 to hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/oldWALs/c2ef38372881%2C33473%2C1733343636517.1733343636639 2024-12-04T20:21:10,304 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/MasterData/oldWALs/c2ef38372881%2C33473%2C1733343636517.1733343636639 to hdfs://localhost:43357/user/jenkins/test-data/cb1ca654-5c12-9108-650e-73f9cbd22c28/oldWALs/c2ef38372881%2C33473%2C1733343636517.1733343636639$masterlocalwal$ 2024-12-04T20:21:10,305 INFO [M:0;c2ef38372881:33473 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:21:10,305 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:21:10,305 INFO [M:0;c2ef38372881:33473 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33473 2024-12-04T20:21:10,305 INFO [M:0;c2ef38372881:33473 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:21:10,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:21:10,407 INFO [M:0;c2ef38372881:33473 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:21:10,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33473-0x100a6e448800000, quorum=127.0.0.1:50038, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:21:10,409 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@73f17350{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:21:10,410 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7da73466{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:21:10,410 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:21:10,410 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1decdda3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:21:10,410 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ebbad67{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,STOPPED} 2024-12-04T20:21:10,412 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:21:10,412 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:21:10,412 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:21:10,412 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1711332990-172.17.0.2-1733343635930 (Datanode Uuid 9f6ab646-fabd-46cc-b732-79c4649a8ae1) service to localhost/127.0.0.1:43357 2024-12-04T20:21:10,412 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data3/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:21:10,412 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data4/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:21:10,413 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:21:10,415 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:10,415 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:10,420 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5aca21ee{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:21:10,421 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@570cb725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:21:10,421 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:21:10,421 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20432799{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:21:10,421 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32c717fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,STOPPED} 2024-12-04T20:21:10,422 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:21:10,422 WARN [BP-1711332990-172.17.0.2-1733343635930 heartbeating to localhost/127.0.0.1:43357 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1711332990-172.17.0.2-1733343635930 (Datanode Uuid 81123f1c-02df-4806-bcc6-1ecb87f7a5bf) service to localhost/127.0.0.1:43357 2024-12-04T20:21:10,423 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data1/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:21:10,423 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/cluster_def1ae4b-df0c-7fb8-6824-4a8cad9da72d/data/data2/current/BP-1711332990-172.17.0.2-1733343635930 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:21:10,423 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:21:10,423 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:21:10,423 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:21:10,430 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1ea36316{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:21:10,430 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d48810f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:21:10,430 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:21:10,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13fdd007{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:21:10,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@421a8f73{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir/,STOPPED} 2024-12-04T20:21:10,437 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:21:10,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T20:21:10,463 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=182 (was 156) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:43357 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43357 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43357 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43357 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43357 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:43357 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43357 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43357 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=60 (was 32) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4465 (was 4614) 2024-12-04T20:21:10,472 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=182, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=60, ProcessCount=11, AvailableMemoryMB=4464 2024-12-04T20:21:10,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:21:10,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.log.dir so I do NOT create it in target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d 2024-12-04T20:21:10,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1114b569-46e1-83d0-b00c-c396ca01b7bb/hadoop.tmp.dir so I do NOT create it in target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d 2024-12-04T20:21:10,472 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47, deleteOnExit=true 2024-12-04T20:21:10,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/test.cache.data in system properties and HBase conf 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:21:10,473 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:21:10,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:21:10,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:21:10,484 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:21:10,528 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:21:10,532 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:21:10,533 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:21:10,533 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:21:10,533 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:21:10,533 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:21:10,534 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@294b1089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:21:10,534 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79c156a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:21:10,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6bac3ffc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/java.io.tmpdir/jetty-localhost-39535-hadoop-hdfs-3_4_1-tests_jar-_-any-4682272743609775191/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:21:10,624 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4a31a089{HTTP/1.1, (http/1.1)}{localhost:39535} 2024-12-04T20:21:10,624 INFO [Time-limited test {}] server.Server(415): Started @182464ms 2024-12-04T20:21:10,634 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:21:10,670 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:21:10,673 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:21:10,673 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:21:10,673 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:21:10,673 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:21:10,674 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@67f9152{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:21:10,674 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ac0122b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:21:10,769 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6dc3ea71{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/java.io.tmpdir/jetty-localhost-42987-hadoop-hdfs-3_4_1-tests_jar-_-any-3747383123841610639/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:21:10,770 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5718d675{HTTP/1.1, (http/1.1)}{localhost:42987} 2024-12-04T20:21:10,770 INFO [Time-limited test {}] server.Server(415): Started @182610ms 2024-12-04T20:21:10,771 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:21:10,803 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:21:10,808 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:21:10,810 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:21:10,810 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:21:10,810 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:21:10,811 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ec454b0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:21:10,811 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39212263{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:21:10,834 WARN [Thread-1652 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/data/data1/current/BP-338094082-172.17.0.2-1733343670495/current, will proceed with Du for space computation calculation, 2024-12-04T20:21:10,835 WARN [Thread-1653 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/data/data2/current/BP-338094082-172.17.0.2-1733343670495/current, will proceed with Du for space computation calculation, 2024-12-04T20:21:10,849 WARN [Thread-1631 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:21:10,852 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe630290c23849192 with lease ID 0xdf34e656010522a4: Processing first storage report for DS-d3e2cffe-aa7b-4b3f-bdf5-3512d0122c8c from datanode DatanodeRegistration(127.0.0.1:39421, datanodeUuid=1a6fdab3-4332-4580-b3b6-68fafb5c707e, infoPort=45121, infoSecurePort=0, ipcPort=44959, storageInfo=lv=-57;cid=testClusterID;nsid=864857372;c=1733343670495) 2024-12-04T20:21:10,852 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe630290c23849192 with lease ID 0xdf34e656010522a4: from storage DS-d3e2cffe-aa7b-4b3f-bdf5-3512d0122c8c node DatanodeRegistration(127.0.0.1:39421, datanodeUuid=1a6fdab3-4332-4580-b3b6-68fafb5c707e, infoPort=45121, infoSecurePort=0, ipcPort=44959, storageInfo=lv=-57;cid=testClusterID;nsid=864857372;c=1733343670495), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:21:10,852 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe630290c23849192 with lease ID 0xdf34e656010522a4: Processing first storage report for DS-628574e8-b0b5-424f-93f7-831d9124e517 from datanode DatanodeRegistration(127.0.0.1:39421, datanodeUuid=1a6fdab3-4332-4580-b3b6-68fafb5c707e, infoPort=45121, infoSecurePort=0, ipcPort=44959, storageInfo=lv=-57;cid=testClusterID;nsid=864857372;c=1733343670495) 2024-12-04T20:21:10,852 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe630290c23849192 with lease ID 0xdf34e656010522a4: from storage DS-628574e8-b0b5-424f-93f7-831d9124e517 node DatanodeRegistration(127.0.0.1:39421, datanodeUuid=1a6fdab3-4332-4580-b3b6-68fafb5c707e, infoPort=45121, infoSecurePort=0, ipcPort=44959, storageInfo=lv=-57;cid=testClusterID;nsid=864857372;c=1733343670495), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:21:10,903 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@18701e65{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/java.io.tmpdir/jetty-localhost-34793-hadoop-hdfs-3_4_1-tests_jar-_-any-5497155815095188833/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:21:10,903 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4dfe8781{HTTP/1.1, (http/1.1)}{localhost:34793} 2024-12-04T20:21:10,904 INFO [Time-limited test {}] server.Server(415): Started @182744ms 2024-12-04T20:21:10,904 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:21:10,978 WARN [Thread-1679 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/data/data4/current/BP-338094082-172.17.0.2-1733343670495/current, will proceed with Du for space computation calculation, 2024-12-04T20:21:10,978 WARN [Thread-1678 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/data/data3/current/BP-338094082-172.17.0.2-1733343670495/current, will proceed with Du for space computation calculation, 2024-12-04T20:21:10,998 WARN [Thread-1667 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:21:11,000 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe3dc556fb4af5df7 with lease ID 0xdf34e656010522a5: Processing first storage report for DS-05d3f6e9-8414-4227-b87c-45753e596da8 from datanode DatanodeRegistration(127.0.0.1:39941, datanodeUuid=a1a53cca-d5a3-4d5c-89ff-ab7d1d772bf2, infoPort=38241, infoSecurePort=0, ipcPort=44423, storageInfo=lv=-57;cid=testClusterID;nsid=864857372;c=1733343670495) 2024-12-04T20:21:11,000 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe3dc556fb4af5df7 with lease ID 0xdf34e656010522a5: from storage DS-05d3f6e9-8414-4227-b87c-45753e596da8 node DatanodeRegistration(127.0.0.1:39941, datanodeUuid=a1a53cca-d5a3-4d5c-89ff-ab7d1d772bf2, infoPort=38241, infoSecurePort=0, ipcPort=44423, storageInfo=lv=-57;cid=testClusterID;nsid=864857372;c=1733343670495), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:21:11,000 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe3dc556fb4af5df7 with lease ID 0xdf34e656010522a5: Processing first storage report for DS-717c3eba-bda8-4886-b2bb-eb7afe09f066 from datanode DatanodeRegistration(127.0.0.1:39941, datanodeUuid=a1a53cca-d5a3-4d5c-89ff-ab7d1d772bf2, infoPort=38241, infoSecurePort=0, ipcPort=44423, storageInfo=lv=-57;cid=testClusterID;nsid=864857372;c=1733343670495) 2024-12-04T20:21:11,001 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe3dc556fb4af5df7 with lease ID 0xdf34e656010522a5: from storage DS-717c3eba-bda8-4886-b2bb-eb7afe09f066 node DatanodeRegistration(127.0.0.1:39941, datanodeUuid=a1a53cca-d5a3-4d5c-89ff-ab7d1d772bf2, infoPort=38241, infoSecurePort=0, ipcPort=44423, storageInfo=lv=-57;cid=testClusterID;nsid=864857372;c=1733343670495), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:21:11,040 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d 2024-12-04T20:21:11,047 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/zookeeper_0, clientPort=63107, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:21:11,049 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63107 2024-12-04T20:21:11,050 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:21:11,051 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:21:11,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:21:11,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:21:11,067 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1 with version=8 2024-12-04T20:21:11,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase-staging 2024-12-04T20:21:11,069 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:21:11,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:21:11,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:21:11,069 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:21:11,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:21:11,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:21:11,069 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:21:11,069 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:21:11,070 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38949 2024-12-04T20:21:11,071 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38949 connecting to ZooKeeper ensemble=127.0.0.1:63107 2024-12-04T20:21:11,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:389490x0, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:21:11,077 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38949-0x100a6e4cf740000 connected 2024-12-04T20:21:11,114 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:21:11,115 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:21:11,117 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:21:11,117 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1, hbase.cluster.distributed=false 2024-12-04T20:21:11,121 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:21:11,125 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38949 2024-12-04T20:21:11,126 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38949 2024-12-04T20:21:11,129 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38949 2024-12-04T20:21:11,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38949 2024-12-04T20:21:11,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38949 2024-12-04T20:21:11,152 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:21:11,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:21:11,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:21:11,153 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:21:11,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:21:11,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:21:11,153 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:21:11,153 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:21:11,154 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37901 2024-12-04T20:21:11,155 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37901 connecting to ZooKeeper ensemble=127.0.0.1:63107 2024-12-04T20:21:11,156 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:21:11,158 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:21:11,167 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379010x0, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:21:11,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:379010x0, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:21:11,167 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37901-0x100a6e4cf740001 connected 2024-12-04T20:21:11,167 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:21:11,168 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:21:11,168 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:21:11,169 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:21:11,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37901 2024-12-04T20:21:11,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37901 2024-12-04T20:21:11,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37901 2024-12-04T20:21:11,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37901 2024-12-04T20:21:11,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37901 2024-12-04T20:21:11,187 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:38949 2024-12-04T20:21:11,187 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,38949,1733343671068 2024-12-04T20:21:11,188 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:21:11,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:21:11,189 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,38949,1733343671068 2024-12-04T20:21:11,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,190 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:21:11,190 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,190 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:21:11,190 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,38949,1733343671068 from backup master directory 2024-12-04T20:21:11,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,38949,1733343671068 2024-12-04T20:21:11,191 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:21:11,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:21:11,191 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:21:11,191 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,38949,1733343671068 2024-12-04T20:21:11,195 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/hbase.id] with ID: 67879a8b-a3b3-4aa5-b710-467c8182f3fe 2024-12-04T20:21:11,195 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/.tmp/hbase.id 2024-12-04T20:21:11,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:21:11,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:21:11,200 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/.tmp/hbase.id]:[hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/hbase.id] 2024-12-04T20:21:11,211 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:21:11,211 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:21:11,213 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-04T20:21:11,214 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:21:11,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:21:11,223 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:21:11,224 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:21:11,224 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:21:11,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:21:11,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:21:11,231 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store 2024-12-04T20:21:11,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:21:11,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:21:11,237 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:21:11,237 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:21:11,237 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:21:11,237 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:21:11,237 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:21:11,237 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:21:11,237 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:21:11,237 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343671237Disabling compacts and flushes for region at 1733343671237Disabling writes for close at 1733343671237Writing region close event to WAL at 1733343671237Closed at 1733343671237 2024-12-04T20:21:11,238 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/.initializing 2024-12-04T20:21:11,238 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/WALs/c2ef38372881,38949,1733343671068 2024-12-04T20:21:11,240 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C38949%2C1733343671068, suffix=, logDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/WALs/c2ef38372881,38949,1733343671068, archiveDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/oldWALs, maxLogs=10 2024-12-04T20:21:11,240 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C38949%2C1733343671068.1733343671240 2024-12-04T20:21:11,244 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/WALs/c2ef38372881,38949,1733343671068/c2ef38372881%2C38949%2C1733343671068.1733343671240 2024-12-04T20:21:11,245 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38241:38241),(127.0.0.1/127.0.0.1:45121:45121)] 2024-12-04T20:21:11,245 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:21:11,245 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:21:11,246 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,246 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:21:11,248 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:21:11,249 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:21:11,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:21:11,251 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:21:11,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:21:11,253 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:21:11,254 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,254 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,255 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,256 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,256 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,257 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:21:11,258 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:21:11,259 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:21:11,260 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743846, jitterRate=-0.05415184795856476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:21:11,260 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733343671246Initializing all the Stores at 1733343671246Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343671246Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343671246Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343671247 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343671247Cleaning up temporary data from old regions at 1733343671256 (+9 ms)Region opened successfully at 1733343671260 (+4 ms) 2024-12-04T20:21:11,261 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:21:11,263 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eae31f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:21:11,264 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:21:11,264 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:21:11,264 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:21:11,264 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:21:11,265 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T20:21:11,265 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T20:21:11,265 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:21:11,267 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:21:11,268 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:21:11,268 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:21:11,269 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:21:11,269 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:21:11,270 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:21:11,270 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:21:11,271 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:21:11,272 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:21:11,272 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:21:11,273 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:21:11,275 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:21:11,275 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:21:11,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:21:11,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,276 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:21:11,276 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,277 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,38949,1733343671068, sessionid=0x100a6e4cf740000, setting cluster-up flag (Was=false) 2024-12-04T20:21:11,278 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,282 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:21:11,282 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,38949,1733343671068 2024-12-04T20:21:11,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,285 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,288 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:21:11,288 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,38949,1733343671068 2024-12-04T20:21:11,290 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:21:11,291 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:21:11,292 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:21:11,292 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:21:11,292 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,38949,1733343671068 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:21:11,294 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:21:11,294 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:21:11,294 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:21:11,294 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:21:11,294 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:21:11,294 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,294 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:21:11,294 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,298 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733343701298 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,299 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:21:11,299 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:21:11,299 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:21:11,300 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:21:11,300 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:21:11,300 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343671300,5,FailOnTimeoutGroup] 2024-12-04T20:21:11,300 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343671300,5,FailOnTimeoutGroup] 2024-12-04T20:21:11,300 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,300 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:21:11,300 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,300 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,301 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,301 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:21:11,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:21:11,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:21:11,315 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:21:11,316 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1 2024-12-04T20:21:11,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:21:11,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:21:11,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:21:11,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:21:11,325 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:21:11,326 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:21:11,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:21:11,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:21:11,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:21:11,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:21:11,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:21:11,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:21:11,341 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740 2024-12-04T20:21:11,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740 2024-12-04T20:21:11,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:21:11,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:21:11,343 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:21:11,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:21:11,351 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:21:11,352 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752206, jitterRate=-0.04352124035358429}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:21:11,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733343671323Initializing all the Stores at 1733343671323Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343671324 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343671324Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343671324Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343671324Cleaning up temporary data from old regions at 1733343671343 (+19 ms)Region opened successfully at 1733343671353 (+10 ms) 2024-12-04T20:21:11,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:21:11,353 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:21:11,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:21:11,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:21:11,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:21:11,354 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:21:11,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343671353Disabling compacts and flushes for region at 1733343671353Disabling writes for close at 1733343671353Writing region close event to WAL at 1733343671353Closed at 1733343671354 (+1 ms) 2024-12-04T20:21:11,355 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:21:11,355 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:21:11,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:21:11,357 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:21:11,358 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:21:11,378 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(746): ClusterId : 67879a8b-a3b3-4aa5-b710-467c8182f3fe 2024-12-04T20:21:11,378 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:21:11,380 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:21:11,380 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:21:11,382 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:21:11,382 DEBUG [RS:0;c2ef38372881:37901 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ed1ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:21:11,399 DEBUG [RS:0;c2ef38372881:37901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:37901 2024-12-04T20:21:11,399 INFO [RS:0;c2ef38372881:37901 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:21:11,399 INFO [RS:0;c2ef38372881:37901 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:21:11,399 DEBUG [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:21:11,400 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,38949,1733343671068 with port=37901, startcode=1733343671152 2024-12-04T20:21:11,400 DEBUG [RS:0;c2ef38372881:37901 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:21:11,402 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33165, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:21:11,403 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38949 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,403 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38949 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,405 DEBUG [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1 2024-12-04T20:21:11,405 DEBUG [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40221 2024-12-04T20:21:11,405 DEBUG [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:21:11,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:21:11,407 DEBUG [RS:0;c2ef38372881:37901 {}] zookeeper.ZKUtil(111): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,407 WARN [RS:0;c2ef38372881:37901 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:21:11,407 INFO [RS:0;c2ef38372881:37901 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:21:11,407 DEBUG [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,407 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,37901,1733343671152] 2024-12-04T20:21:11,411 INFO [RS:0;c2ef38372881:37901 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:21:11,412 INFO [RS:0;c2ef38372881:37901 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:21:11,413 INFO [RS:0;c2ef38372881:37901 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:21:11,413 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,413 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:21:11,414 INFO [RS:0;c2ef38372881:37901 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:21:11,414 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,414 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,414 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,414 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,414 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,414 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,414 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:21:11,414 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,414 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,415 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,415 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,415 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,415 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:21:11,415 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:21:11,415 DEBUG [RS:0;c2ef38372881:37901 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:21:11,415 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:11,415 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:11,420 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,420 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,420 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,421 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,421 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,421 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37901,1733343671152-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:21:11,441 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:21:11,441 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37901,1733343671152-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,441 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,441 INFO [RS:0;c2ef38372881:37901 {}] regionserver.Replication(171): c2ef38372881,37901,1733343671152 started 2024-12-04T20:21:11,461 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,461 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,37901,1733343671152, RpcServer on c2ef38372881/172.17.0.2:37901, sessionid=0x100a6e4cf740001 2024-12-04T20:21:11,461 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:21:11,461 DEBUG [RS:0;c2ef38372881:37901 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,461 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,37901,1733343671152' 2024-12-04T20:21:11,461 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:21:11,462 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:21:11,462 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:21:11,462 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:21:11,462 DEBUG [RS:0;c2ef38372881:37901 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,462 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,37901,1733343671152' 2024-12-04T20:21:11,462 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:21:11,463 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:21:11,463 DEBUG [RS:0;c2ef38372881:37901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:21:11,463 INFO [RS:0;c2ef38372881:37901 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:21:11,463 INFO [RS:0;c2ef38372881:37901 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:21:11,508 WARN [c2ef38372881:38949 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T20:21:11,566 INFO [RS:0;c2ef38372881:37901 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C37901%2C1733343671152, suffix=, logDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152, archiveDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/oldWALs, maxLogs=32 2024-12-04T20:21:11,567 INFO [RS:0;c2ef38372881:37901 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37901%2C1733343671152.1733343671567 2024-12-04T20:21:11,575 INFO [RS:0;c2ef38372881:37901 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343671567 2024-12-04T20:21:11,578 DEBUG [RS:0;c2ef38372881:37901 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45121:45121),(127.0.0.1/127.0.0.1:38241:38241)] 2024-12-04T20:21:11,758 DEBUG [c2ef38372881:38949 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T20:21:11,759 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,760 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,37901,1733343671152, state=OPENING 2024-12-04T20:21:11,761 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:21:11,762 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:21:11,762 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:21:11,763 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:21:11,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,37901,1733343671152}] 2024-12-04T20:21:11,763 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:21:11,916 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:21:11,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43299, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:21:11,923 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:21:11,924 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:21:11,927 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C37901%2C1733343671152.meta, suffix=.meta, logDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152, archiveDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/oldWALs, maxLogs=32 2024-12-04T20:21:11,927 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37901%2C1733343671152.meta.1733343671927.meta 2024-12-04T20:21:11,934 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.meta.1733343671927.meta 2024-12-04T20:21:11,941 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45121:45121),(127.0.0.1/127.0.0.1:38241:38241)] 2024-12-04T20:21:11,941 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:21:11,942 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:21:11,942 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:21:11,942 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:21:11,942 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:21:11,942 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:21:11,942 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:21:11,942 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:21:11,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:21:11,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:21:11,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:21:11,946 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:21:11,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:21:11,947 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:21:11,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:21:11,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:21:11,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:11,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:21:11,949 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:21:11,949 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740 2024-12-04T20:21:11,950 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740 2024-12-04T20:21:11,951 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:21:11,951 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:21:11,952 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:21:11,953 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:21:11,953 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=870340, jitterRate=0.10669535398483276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:21:11,953 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:21:11,954 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733343671942Writing region info on filesystem at 1733343671942Initializing all the Stores at 1733343671943 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343671943Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343671943Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343671943Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343671943Cleaning up temporary data from old regions at 1733343671951 (+8 ms)Running coprocessor post-open hooks at 1733343671953 (+2 ms)Region opened successfully at 1733343671954 (+1 ms) 2024-12-04T20:21:11,955 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733343671915 2024-12-04T20:21:11,958 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:21:11,958 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:21:11,960 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,961 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,37901,1733343671152, state=OPEN 2024-12-04T20:21:11,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:21:11,964 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:21:11,964 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,37901,1733343671152 2024-12-04T20:21:11,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:21:11,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:21:11,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:21:11,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,37901,1733343671152 in 201 msec 2024-12-04T20:21:11,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:21:11,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 612 msec 2024-12-04T20:21:11,972 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:21:11,972 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:21:11,973 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:21:11,973 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,37901,1733343671152, seqNum=-1] 2024-12-04T20:21:11,974 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:21:11,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52787, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:21:11,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 691 msec 2024-12-04T20:21:11,983 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733343671983, completionTime=-1 2024-12-04T20:21:11,983 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T20:21:11,983 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733343731986 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733343791986 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,38949,1733343671068-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,38949,1733343671068-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,38949,1733343671068-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:38949, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,986 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:11,989 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:21:11,991 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.800sec 2024-12-04T20:21:11,992 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:21:11,992 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:21:11,992 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:21:11,992 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:21:11,992 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:21:11,992 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,38949,1733343671068-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:21:11,992 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,38949,1733343671068-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:21:11,994 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:21:11,995 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:21:11,995 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,38949,1733343671068-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:12,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7009eb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:21:12,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,38949,-1 for getting cluster id 2024-12-04T20:21:12,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:21:12,082 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67879a8b-a3b3-4aa5-b710-467c8182f3fe' 2024-12-04T20:21:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:21:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67879a8b-a3b3-4aa5-b710-467c8182f3fe" 2024-12-04T20:21:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8889fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:21:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,38949,-1] 2024-12-04T20:21:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:21:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:21:12,085 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:21:12,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5daea1f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:21:12,087 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:21:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,37901,1733343671152, seqNum=-1] 2024-12-04T20:21:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:21:12,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:21:12,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,38949,1733343671068 2024-12-04T20:21:12,092 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:21:12,096 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T20:21:12,097 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:21:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,38949,1733343671068 2024-12-04T20:21:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32b03fb8 2024-12-04T20:21:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:21:12,099 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:21:12,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T20:21:12,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T20:21:12,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:21:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:12,103 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:21:12,104 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:12,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-12-04T20:21:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:21:12,105 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:21:12,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741835_1011 (size=405) 2024-12-04T20:21:12,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741835_1011 (size=405) 2024-12-04T20:21:12,113 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a5092b77e6e0d374bdb1bdf2211b335, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1 2024-12-04T20:21:12,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741836_1012 (size=88) 2024-12-04T20:21:12,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741836_1012 (size=88) 2024-12-04T20:21:12,119 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:21:12,120 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 1a5092b77e6e0d374bdb1bdf2211b335, disabling compactions & flushes 2024-12-04T20:21:12,120 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:12,120 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:12,120 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. after waiting 0 ms 2024-12-04T20:21:12,120 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:12,120 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:12,120 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a5092b77e6e0d374bdb1bdf2211b335: Waiting for close lock at 1733343672120Disabling compacts and flushes for region at 1733343672120Disabling writes for close at 1733343672120Writing region close event to WAL at 1733343672120Closed at 1733343672120 2024-12-04T20:21:12,121 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:21:12,121 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733343672121"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733343672121"}]},"ts":"1733343672121"} 2024-12-04T20:21:12,124 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:21:12,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:21:12,125 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343672125"}]},"ts":"1733343672125"} 2024-12-04T20:21:12,128 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-12-04T20:21:12,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1a5092b77e6e0d374bdb1bdf2211b335, ASSIGN}] 2024-12-04T20:21:12,129 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1a5092b77e6e0d374bdb1bdf2211b335, ASSIGN 2024-12-04T20:21:12,131 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1a5092b77e6e0d374bdb1bdf2211b335, ASSIGN; state=OFFLINE, location=c2ef38372881,37901,1733343671152; forceNewPlan=false, retain=false 2024-12-04T20:21:12,281 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a5092b77e6e0d374bdb1bdf2211b335, regionState=OPENING, regionLocation=c2ef38372881,37901,1733343671152 2024-12-04T20:21:12,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1a5092b77e6e0d374bdb1bdf2211b335, ASSIGN because future has completed 2024-12-04T20:21:12,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a5092b77e6e0d374bdb1bdf2211b335, server=c2ef38372881,37901,1733343671152}] 2024-12-04T20:21:12,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:12,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:12,446 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:12,446 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a5092b77e6e0d374bdb1bdf2211b335, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:21:12,446 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,447 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:21:12,447 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,447 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,449 INFO [StoreOpener-1a5092b77e6e0d374bdb1bdf2211b335-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,451 INFO [StoreOpener-1a5092b77e6e0d374bdb1bdf2211b335-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5092b77e6e0d374bdb1bdf2211b335 columnFamilyName info 2024-12-04T20:21:12,451 DEBUG [StoreOpener-1a5092b77e6e0d374bdb1bdf2211b335-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:21:12,452 INFO [StoreOpener-1a5092b77e6e0d374bdb1bdf2211b335-1 {}] regionserver.HStore(327): Store=1a5092b77e6e0d374bdb1bdf2211b335/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:21:12,452 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,453 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,454 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,454 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,454 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,457 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,460 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:21:12,461 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1a5092b77e6e0d374bdb1bdf2211b335; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=733703, jitterRate=-0.06704843044281006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:21:12,461 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:21:12,462 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1a5092b77e6e0d374bdb1bdf2211b335: Running coprocessor pre-open hook at 1733343672447Writing region info on filesystem at 1733343672447Initializing all the Stores at 1733343672448 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343672448Cleaning up temporary data from old regions at 1733343672454 (+6 ms)Running coprocessor post-open hooks at 1733343672461 (+7 ms)Region opened successfully at 1733343672462 (+1 ms) 2024-12-04T20:21:12,463 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335., pid=6, masterSystemTime=1733343672439 2024-12-04T20:21:12,465 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:12,465 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:12,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a5092b77e6e0d374bdb1bdf2211b335, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,37901,1733343671152 2024-12-04T20:21:12,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a5092b77e6e0d374bdb1bdf2211b335, server=c2ef38372881,37901,1733343671152 because future has completed 2024-12-04T20:21:12,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T20:21:12,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a5092b77e6e0d374bdb1bdf2211b335, server=c2ef38372881,37901,1733343671152 in 183 msec 2024-12-04T20:21:12,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T20:21:12,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1a5092b77e6e0d374bdb1bdf2211b335, ASSIGN in 343 msec 2024-12-04T20:21:12,475 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:21:12,475 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343672475"}]},"ts":"1733343672475"} 2024-12-04T20:21:12,477 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-12-04T20:21:12,478 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:21:12,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 378 msec 2024-12-04T20:21:12,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:21:12,620 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T20:21:12,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:21:12,620 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T20:21:12,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:12,620 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T20:21:13,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:13,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:14,419 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:14,419 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:15,420 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:15,420 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:16,420 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:16,420 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:17,421 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:17,421 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:17,445 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T20:21:17,446 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,446 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,446 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,446 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,446 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,447 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,463 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,463 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,463 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,465 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:21:17,470 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T20:21:17,470 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-12-04T20:21:18,423 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:18,423 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:19,424 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:19,424 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:20,425 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:20,425 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:21,426 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:21,426 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:21:22,123 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T20:21:22,123 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-12-04T20:21:22,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:22,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335., hostname=c2ef38372881,37901,1733343671152, seqNum=2] 2024-12-04T20:21:22,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:22,147 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:21:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:21:22,149 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:21:22,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:21:22,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37901 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T20:21:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:22,312 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1a5092b77e6e0d374bdb1bdf2211b335 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T20:21:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/1fc808feb9bb4279ac88692ce49bb0ab is 1080, key is row0001/info:/1733343682135/Put/seqid=0 2024-12-04T20:21:22,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741837_1013 (size=6033) 2024-12-04T20:21:22,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741837_1013 (size=6033) 2024-12-04T20:21:22,339 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/1fc808feb9bb4279ac88692ce49bb0ab 2024-12-04T20:21:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/1fc808feb9bb4279ac88692ce49bb0ab as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/1fc808feb9bb4279ac88692ce49bb0ab 2024-12-04T20:21:22,352 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/1fc808feb9bb4279ac88692ce49bb0ab, entries=1, sequenceid=5, filesize=5.9 K 2024-12-04T20:21:22,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1a5092b77e6e0d374bdb1bdf2211b335 in 41ms, sequenceid=5, compaction requested=false 2024-12-04T20:21:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1a5092b77e6e0d374bdb1bdf2211b335: 2024-12-04T20:21:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-04T20:21:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-04T20:21:22,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T20:21:22,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 207 msec 2024-12-04T20:21:22,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 218 msec 2024-12-04T20:21:22,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:22,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:23,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:23,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:24,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:24,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:25,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:25,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:26,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:26,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:27,431 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:27,431 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:28,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:28,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:29,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:29,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:30,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:30,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:31,436 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:31,436 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T20:21:32,213 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T20:21:32,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:21:32,223 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:21:32,225 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:21:32,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:21:32,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37901 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T20:21:32,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:32,382 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1a5092b77e6e0d374bdb1bdf2211b335 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T20:21:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/70a9ce15e29248fc93eeb98673f4bdb4 is 1080, key is row0002/info:/1733343692215/Put/seqid=0 2024-12-04T20:21:32,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741838_1014 (size=6033) 2024-12-04T20:21:32,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741838_1014 (size=6033) 2024-12-04T20:21:32,398 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/70a9ce15e29248fc93eeb98673f4bdb4 2024-12-04T20:21:32,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/70a9ce15e29248fc93eeb98673f4bdb4 as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/70a9ce15e29248fc93eeb98673f4bdb4 2024-12-04T20:21:32,412 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/70a9ce15e29248fc93eeb98673f4bdb4, entries=1, sequenceid=9, filesize=5.9 K 2024-12-04T20:21:32,413 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1a5092b77e6e0d374bdb1bdf2211b335 in 31ms, sequenceid=9, compaction requested=false 2024-12-04T20:21:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1a5092b77e6e0d374bdb1bdf2211b335: 2024-12-04T20:21:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-04T20:21:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-04T20:21:32,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T20:21:32,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 189 msec 2024-12-04T20:21:32,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 198 msec 2024-12-04T20:21:32,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:32,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:33,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:33,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:34,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:34,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:35,439 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:35,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:36,440 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:36,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:37,440 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:37,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 after 68067ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:21:37,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:37,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta after 68054ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:21:38,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:38,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:39,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:39,443 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:40,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:40,443 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:41,039 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:21:41,443 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:41,444 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T20:21:42,291 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T20:21:42,294 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37901%2C1733343671152.1733343702294 2024-12-04T20:21:42,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:42,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:42,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:42,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:42,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:42,303 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343671567 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343702294 2024-12-04T20:21:42,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741833_1009 (size=5546) 2024-12-04T20:21:42,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741833_1009 (size=5546) 2024-12-04T20:21:42,307 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45121:45121),(127.0.0.1/127.0.0.1:38241:38241)] 2024-12-04T20:21:42,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:21:42,310 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:21:42,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:21:42,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:21:42,444 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:42,444 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:42,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37901 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T20:21:42,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:42,465 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1a5092b77e6e0d374bdb1bdf2211b335 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T20:21:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/8718eea7967e47cda35a971dcfcfd59e is 1080, key is row0003/info:/1733343702292/Put/seqid=0 2024-12-04T20:21:42,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741840_1016 (size=6033) 2024-12-04T20:21:42,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741840_1016 (size=6033) 2024-12-04T20:21:42,474 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/8718eea7967e47cda35a971dcfcfd59e 2024-12-04T20:21:42,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/8718eea7967e47cda35a971dcfcfd59e as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/8718eea7967e47cda35a971dcfcfd59e 2024-12-04T20:21:42,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/8718eea7967e47cda35a971dcfcfd59e, entries=1, sequenceid=13, filesize=5.9 K 2024-12-04T20:21:42,487 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1a5092b77e6e0d374bdb1bdf2211b335 in 22ms, sequenceid=13, compaction requested=true 2024-12-04T20:21:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1a5092b77e6e0d374bdb1bdf2211b335: 2024-12-04T20:21:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-04T20:21:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-04T20:21:42,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T20:21:42,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec 2024-12-04T20:21:42,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec 2024-12-04T20:21:43,445 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:43,446 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:44,445 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:44,446 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:45,446 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:45,447 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:46,447 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:46,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:47,447 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:47,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:48,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:48,449 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:49,449 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:49,449 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:50,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:50,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:51,451 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:51,451 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:52,056 INFO [master/c2ef38372881:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T20:21:52,056 INFO [master/c2ef38372881:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T20:21:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T20:21:52,353 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T20:21:52,353 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:21:52,357 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:21:52,357 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1a5092b77e6e0d374bdb1bdf2211b335/info is initiating minor compaction (all files) 2024-12-04T20:21:52,357 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:21:52,358 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:21:52,358 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1a5092b77e6e0d374bdb1bdf2211b335/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:52,358 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/1fc808feb9bb4279ac88692ce49bb0ab, hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/70a9ce15e29248fc93eeb98673f4bdb4, hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/8718eea7967e47cda35a971dcfcfd59e] into tmpdir=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp, totalSize=17.7 K 2024-12-04T20:21:52,359 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1fc808feb9bb4279ac88692ce49bb0ab, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733343682135 2024-12-04T20:21:52,360 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 70a9ce15e29248fc93eeb98673f4bdb4, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733343692215 2024-12-04T20:21:52,361 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8718eea7967e47cda35a971dcfcfd59e, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733343702292 2024-12-04T20:21:52,378 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1a5092b77e6e0d374bdb1bdf2211b335#info#compaction#46 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:21:52,379 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/df3aea4161244ada8d68fb7802adc286 is 1080, key is row0001/info:/1733343682135/Put/seqid=0 2024-12-04T20:21:52,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741841_1017 (size=8296) 2024-12-04T20:21:52,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741841_1017 (size=8296) 2024-12-04T20:21:52,389 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/df3aea4161244ada8d68fb7802adc286 as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/df3aea4161244ada8d68fb7802adc286 2024-12-04T20:21:52,397 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a5092b77e6e0d374bdb1bdf2211b335/info of 1a5092b77e6e0d374bdb1bdf2211b335 into df3aea4161244ada8d68fb7802adc286(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:21:52,397 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1a5092b77e6e0d374bdb1bdf2211b335: 2024-12-04T20:21:52,400 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37901%2C1733343671152.1733343712400 2024-12-04T20:21:52,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:52,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:52,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:52,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:52,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:21:52,406 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343702294 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343712400 2024-12-04T20:21:52,407 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45121:45121),(127.0.0.1/127.0.0.1:38241:38241)] 2024-12-04T20:21:52,407 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343702294 is not closed yet, will try archiving it next time 2024-12-04T20:21:52,408 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343671567 to hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/oldWALs/c2ef38372881%2C37901%2C1733343671152.1733343671567 2024-12-04T20:21:52,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741839_1015 (size=2520) 2024-12-04T20:21:52,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741839_1015 (size=2520) 2024-12-04T20:21:52,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:21:52,411 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T20:21:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T20:21:52,411 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T20:21:52,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T20:21:52,451 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:52,451 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:52,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37901 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T20:21:52,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:52,566 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1a5092b77e6e0d374bdb1bdf2211b335 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T20:21:52,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/a001cffe12544939b1af6644e7b5606b is 1080, key is row0000/info:/1733343712398/Put/seqid=0 2024-12-04T20:21:52,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741843_1019 (size=6033) 2024-12-04T20:21:52,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741843_1019 (size=6033) 2024-12-04T20:21:52,581 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/a001cffe12544939b1af6644e7b5606b 2024-12-04T20:21:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/a001cffe12544939b1af6644e7b5606b as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/a001cffe12544939b1af6644e7b5606b 2024-12-04T20:21:52,592 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/a001cffe12544939b1af6644e7b5606b, entries=1, sequenceid=18, filesize=5.9 K 2024-12-04T20:21:52,593 INFO [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1a5092b77e6e0d374bdb1bdf2211b335 in 28ms, sequenceid=18, compaction requested=false 2024-12-04T20:21:52,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1a5092b77e6e0d374bdb1bdf2211b335: 2024-12-04T20:21:52,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:21:52,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-04T20:21:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-04T20:21:52,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T20:21:52,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 183 msec 2024-12-04T20:21:52,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 188 msec 2024-12-04T20:21:53,452 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:53,452 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:54,453 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:54,453 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:55,454 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:55,454 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:56,454 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:56,454 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:57,447 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1a5092b77e6e0d374bdb1bdf2211b335, had cached 0 bytes from a total of 14329 2024-12-04T20:21:57,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:57,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:58,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:58,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:59,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:21:59,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:00,458 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:00,458 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:01,458 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:01,458 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T20:22:02,423 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T20:22:02,429 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C37901%2C1733343671152.1733343722429 2024-12-04T20:22:02,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,440 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343712400 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343722429 2024-12-04T20:22:02,441 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38241:38241),(127.0.0.1/127.0.0.1:45121:45121)] 2024-12-04T20:22:02,442 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343712400 is not closed yet, will try archiving it next time 2024-12-04T20:22:02,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:22:02,442 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/WALs/c2ef38372881,37901,1733343671152/c2ef38372881%2C37901%2C1733343671152.1733343702294 to hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/oldWALs/c2ef38372881%2C37901%2C1733343671152.1733343702294 2024-12-04T20:22:02,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:22:02,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:02,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:02,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:02,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:22:02,443 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:22:02,443 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1072930910, stopped=false 2024-12-04T20:22:02,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741842_1018 (size=2026) 2024-12-04T20:22:02,443 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,38949,1733343671068 2024-12-04T20:22:02,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741842_1018 (size=2026) 2024-12-04T20:22:02,444 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:02,444 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:02,444 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:22:02,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:02,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:02,444 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:22:02,445 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:02,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:02,445 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:02,445 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,37901,1733343671152' ***** 2024-12-04T20:22:02,445 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:22:02,445 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:22:02,446 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,37901,1733343671152 2024-12-04T20:22:02,446 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:37901. 2024-12-04T20:22:02,446 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a5092b77e6e0d374bdb1bdf2211b335, disabling compactions & flushes 2024-12-04T20:22:02,446 DEBUG [RS:0;c2ef38372881:37901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:02,446 DEBUG [RS:0;c2ef38372881:37901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:02,446 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:22:02,446 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:22:02,446 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. after waiting 0 ms 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:22:02,446 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:22:02,446 INFO [RS:0;c2ef38372881:37901 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:22:02,447 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:22:02,447 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1a5092b77e6e0d374bdb1bdf2211b335 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T20:22:02,447 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T20:22:02,447 DEBUG [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1325): Online Regions={1a5092b77e6e0d374bdb1bdf2211b335=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T20:22:02,447 DEBUG [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a5092b77e6e0d374bdb1bdf2211b335 2024-12-04T20:22:02,447 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:22:02,447 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:22:02,447 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:22:02,447 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:22:02,447 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:22:02,447 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-12-04T20:22:02,453 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/6f2130d210d843f4b15ff99eee3463b0 is 1080, key is row0001/info:/1733343722425/Put/seqid=0 2024-12-04T20:22:02,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741845_1021 (size=6033) 2024-12-04T20:22:02,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741845_1021 (size=6033) 2024-12-04T20:22:02,459 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/6f2130d210d843f4b15ff99eee3463b0 2024-12-04T20:22:02,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:02,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:02,465 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/.tmp/info/6f2130d210d843f4b15ff99eee3463b0 as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/6f2130d210d843f4b15ff99eee3463b0 2024-12-04T20:22:02,466 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/info/af05b6d333684ed4918ed8e7152c2ddd is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335./info:regioninfo/1733343672465/Put/seqid=0 2024-12-04T20:22:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741846_1022 (size=7308) 2024-12-04T20:22:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741846_1022 (size=7308) 2024-12-04T20:22:02,471 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/info/af05b6d333684ed4918ed8e7152c2ddd 2024-12-04T20:22:02,471 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/6f2130d210d843f4b15ff99eee3463b0, entries=1, sequenceid=22, filesize=5.9 K 2024-12-04T20:22:02,478 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1a5092b77e6e0d374bdb1bdf2211b335 in 31ms, sequenceid=22, compaction requested=true 2024-12-04T20:22:02,478 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/1fc808feb9bb4279ac88692ce49bb0ab, hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/70a9ce15e29248fc93eeb98673f4bdb4, hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/8718eea7967e47cda35a971dcfcfd59e] to archive 2024-12-04T20:22:02,479 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:22:02,480 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/1fc808feb9bb4279ac88692ce49bb0ab to hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/1fc808feb9bb4279ac88692ce49bb0ab 2024-12-04T20:22:02,481 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/70a9ce15e29248fc93eeb98673f4bdb4 to hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/70a9ce15e29248fc93eeb98673f4bdb4 2024-12-04T20:22:02,482 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/8718eea7967e47cda35a971dcfcfd59e to hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/info/8718eea7967e47cda35a971dcfcfd59e 2024-12-04T20:22:02,483 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c2ef38372881:38949 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T20:22:02,483 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [1fc808feb9bb4279ac88692ce49bb0ab=6033, 70a9ce15e29248fc93eeb98673f4bdb4=6033, 8718eea7967e47cda35a971dcfcfd59e=6033] 2024-12-04T20:22:02,487 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1a5092b77e6e0d374bdb1bdf2211b335/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-12-04T20:22:02,487 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:22:02,488 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a5092b77e6e0d374bdb1bdf2211b335: Waiting for close lock at 1733343722446Running coprocessor pre-close hooks at 1733343722446Disabling compacts and flushes for region at 1733343722446Disabling writes for close at 1733343722446Obtaining lock to block concurrent updates at 1733343722447 (+1 ms)Preparing flush snapshotting stores in 1a5092b77e6e0d374bdb1bdf2211b335 at 1733343722447Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733343722447Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. at 1733343722448 (+1 ms)Flushing 1a5092b77e6e0d374bdb1bdf2211b335/info: creating writer at 1733343722448Flushing 1a5092b77e6e0d374bdb1bdf2211b335/info: appending metadata at 1733343722452 (+4 ms)Flushing 1a5092b77e6e0d374bdb1bdf2211b335/info: closing flushed file at 1733343722453 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@761299c: reopening flushed file at 1733343722465 (+12 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1a5092b77e6e0d374bdb1bdf2211b335 in 31ms, sequenceid=22, compaction requested=true at 1733343722478 (+13 ms)Writing region close event to WAL at 1733343722483 (+5 ms)Running coprocessor post-close hooks at 1733343722487 (+4 ms)Closed at 1733343722487 2024-12-04T20:22:02,488 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733343672100.1a5092b77e6e0d374bdb1bdf2211b335. 2024-12-04T20:22:02,495 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/ns/624daf1b1dfb4a7190cc2559738b3898 is 43, key is default/ns:d/1733343671976/Put/seqid=0 2024-12-04T20:22:02,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741847_1023 (size=5153) 2024-12-04T20:22:02,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741847_1023 (size=5153) 2024-12-04T20:22:02,500 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/ns/624daf1b1dfb4a7190cc2559738b3898 2024-12-04T20:22:02,518 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/table/31d2ea9a66c7482e98eaec04b24cec21 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733343672475/Put/seqid=0 2024-12-04T20:22:02,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741848_1024 (size=5508) 2024-12-04T20:22:02,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741848_1024 (size=5508) 2024-12-04T20:22:02,522 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/table/31d2ea9a66c7482e98eaec04b24cec21 2024-12-04T20:22:02,527 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/info/af05b6d333684ed4918ed8e7152c2ddd as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/info/af05b6d333684ed4918ed8e7152c2ddd 2024-12-04T20:22:02,532 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/info/af05b6d333684ed4918ed8e7152c2ddd, entries=10, sequenceid=11, filesize=7.1 K 2024-12-04T20:22:02,533 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/ns/624daf1b1dfb4a7190cc2559738b3898 as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/ns/624daf1b1dfb4a7190cc2559738b3898 2024-12-04T20:22:02,540 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/ns/624daf1b1dfb4a7190cc2559738b3898, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T20:22:02,541 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/.tmp/table/31d2ea9a66c7482e98eaec04b24cec21 as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/table/31d2ea9a66c7482e98eaec04b24cec21 2024-12-04T20:22:02,546 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/table/31d2ea9a66c7482e98eaec04b24cec21, entries=2, sequenceid=11, filesize=5.4 K 2024-12-04T20:22:02,547 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=11, compaction requested=false 2024-12-04T20:22:02,552 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T20:22:02,552 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:22:02,552 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:22:02,552 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343722447Running coprocessor pre-close hooks at 1733343722447Disabling compacts and flushes for region at 1733343722447Disabling writes for close at 1733343722447Obtaining lock to block concurrent updates at 1733343722447Preparing flush snapshotting stores in 1588230740 at 1733343722447Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733343722448 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733343722448Flushing 1588230740/info: creating writer at 1733343722449 (+1 ms)Flushing 1588230740/info: appending metadata at 1733343722465 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733343722466 (+1 ms)Flushing 1588230740/ns: creating writer at 1733343722476 (+10 ms)Flushing 1588230740/ns: appending metadata at 1733343722495 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733343722495Flushing 1588230740/table: creating writer at 1733343722504 (+9 ms)Flushing 1588230740/table: appending metadata at 1733343722517 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733343722517Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1eea5f9c: reopening flushed file at 1733343722526 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e2e5dcf: reopening flushed file at 1733343722532 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d6fedde: reopening flushed file at 1733343722540 (+8 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=11, compaction requested=false at 1733343722547 (+7 ms)Writing region close event to WAL at 1733343722548 (+1 ms)Running coprocessor post-close hooks at 1733343722552 (+4 ms)Closed at 1733343722552 2024-12-04T20:22:02,552 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:22:02,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:22:02,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:22:02,621 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T20:22:02,647 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,37901,1733343671152; all regions closed. 2024-12-04T20:22:02,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741834_1010 (size=3306) 2024-12-04T20:22:02,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741834_1010 (size=3306) 2024-12-04T20:22:02,659 DEBUG [RS:0;c2ef38372881:37901 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/oldWALs 2024-12-04T20:22:02,659 INFO [RS:0;c2ef38372881:37901 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C37901%2C1733343671152.meta:.meta(num 1733343671927) 2024-12-04T20:22:02,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741844_1020 (size=1252) 2024-12-04T20:22:02,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741844_1020 (size=1252) 2024-12-04T20:22:02,666 DEBUG [RS:0;c2ef38372881:37901 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/oldWALs 2024-12-04T20:22:02,666 INFO [RS:0;c2ef38372881:37901 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C37901%2C1733343671152:(num 1733343722429) 2024-12-04T20:22:02,666 DEBUG [RS:0;c2ef38372881:37901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:02,666 INFO [RS:0;c2ef38372881:37901 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:22:02,666 INFO [RS:0;c2ef38372881:37901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:22:02,666 INFO [RS:0;c2ef38372881:37901 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T20:22:02,666 INFO [RS:0;c2ef38372881:37901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:22:02,666 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:22:02,666 INFO [RS:0;c2ef38372881:37901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37901 2024-12-04T20:22:02,668 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,37901,1733343671152 2024-12-04T20:22:02,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:22:02,668 INFO [RS:0;c2ef38372881:37901 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:22:02,669 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,37901,1733343671152] 2024-12-04T20:22:02,670 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,37901,1733343671152 already deleted, retry=false 2024-12-04T20:22:02,670 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,37901,1733343671152 expired; onlineServers=0 2024-12-04T20:22:02,670 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,38949,1733343671068' ***** 2024-12-04T20:22:02,670 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:22:02,670 INFO [M:0;c2ef38372881:38949 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:22:02,670 INFO [M:0;c2ef38372881:38949 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:22:02,670 DEBUG [M:0;c2ef38372881:38949 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:22:02,670 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:22:02,670 DEBUG [M:0;c2ef38372881:38949 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:22:02,670 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343671300 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343671300,5,FailOnTimeoutGroup] 2024-12-04T20:22:02,670 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343671300 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343671300,5,FailOnTimeoutGroup] 2024-12-04T20:22:02,671 INFO [M:0;c2ef38372881:38949 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:22:02,671 INFO [M:0;c2ef38372881:38949 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:22:02,671 DEBUG [M:0;c2ef38372881:38949 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:22:02,671 INFO [M:0;c2ef38372881:38949 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:22:02,671 INFO [M:0;c2ef38372881:38949 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:22:02,671 INFO [M:0;c2ef38372881:38949 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:22:02,671 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:22:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:22:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:02,672 DEBUG [M:0;c2ef38372881:38949 {}] zookeeper.ZKUtil(347): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:22:02,672 WARN [M:0;c2ef38372881:38949 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:22:02,673 INFO [M:0;c2ef38372881:38949 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/.lastflushedseqids 2024-12-04T20:22:02,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741849_1025 (size=130) 2024-12-04T20:22:02,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741849_1025 (size=130) 2024-12-04T20:22:02,678 INFO [M:0;c2ef38372881:38949 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:22:02,678 INFO [M:0;c2ef38372881:38949 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:22:02,678 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:22:02,678 INFO [M:0;c2ef38372881:38949 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:02,678 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:02,678 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:22:02,678 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:02,678 INFO [M:0;c2ef38372881:38949 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.56 KB heapSize=54.94 KB 2024-12-04T20:22:02,694 DEBUG [M:0;c2ef38372881:38949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/04b5d2b919fe4999b042aa70d3793e28 is 82, key is hbase:meta,,1/info:regioninfo/1733343671959/Put/seqid=0 2024-12-04T20:22:02,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741850_1026 (size=5672) 2024-12-04T20:22:02,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741850_1026 (size=5672) 2024-12-04T20:22:02,699 INFO [M:0;c2ef38372881:38949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/04b5d2b919fe4999b042aa70d3793e28 2024-12-04T20:22:02,717 DEBUG [M:0;c2ef38372881:38949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4276000c803249bbbea9b298da46b5ca is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733343672479/Put/seqid=0 2024-12-04T20:22:02,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741851_1027 (size=7819) 2024-12-04T20:22:02,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741851_1027 (size=7819) 2024-12-04T20:22:02,722 INFO [M:0;c2ef38372881:38949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.95 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4276000c803249bbbea9b298da46b5ca 2024-12-04T20:22:02,727 INFO [M:0;c2ef38372881:38949 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4276000c803249bbbea9b298da46b5ca 2024-12-04T20:22:02,740 DEBUG [M:0;c2ef38372881:38949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c20f09939029475fb9ebd7baaf4fb0d0 is 69, key is c2ef38372881,37901,1733343671152/rs:state/1733343671403/Put/seqid=0 2024-12-04T20:22:02,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741852_1028 (size=5156) 2024-12-04T20:22:02,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741852_1028 (size=5156) 2024-12-04T20:22:02,745 INFO [M:0;c2ef38372881:38949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c20f09939029475fb9ebd7baaf4fb0d0 2024-12-04T20:22:02,762 DEBUG [M:0;c2ef38372881:38949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd301b3a86c84966b04544984847dda9 is 52, key is load_balancer_on/state:d/1733343672095/Put/seqid=0 2024-12-04T20:22:02,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741853_1029 (size=5056) 2024-12-04T20:22:02,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741853_1029 (size=5056) 2024-12-04T20:22:02,767 INFO [M:0;c2ef38372881:38949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd301b3a86c84966b04544984847dda9 2024-12-04T20:22:02,769 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:02,769 INFO [RS:0;c2ef38372881:37901 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:22:02,769 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37901-0x100a6e4cf740001, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:02,769 INFO [RS:0;c2ef38372881:37901 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,37901,1733343671152; zookeeper connection closed. 2024-12-04T20:22:02,769 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6ec8c398 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6ec8c398 2024-12-04T20:22:02,769 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T20:22:02,772 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/04b5d2b919fe4999b042aa70d3793e28 as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/04b5d2b919fe4999b042aa70d3793e28 2024-12-04T20:22:02,777 INFO [M:0;c2ef38372881:38949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/04b5d2b919fe4999b042aa70d3793e28, entries=8, sequenceid=121, filesize=5.5 K 2024-12-04T20:22:02,778 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4276000c803249bbbea9b298da46b5ca as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4276000c803249bbbea9b298da46b5ca 2024-12-04T20:22:02,784 INFO [M:0;c2ef38372881:38949 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4276000c803249bbbea9b298da46b5ca 2024-12-04T20:22:02,785 INFO [M:0;c2ef38372881:38949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4276000c803249bbbea9b298da46b5ca, entries=14, sequenceid=121, filesize=7.6 K 2024-12-04T20:22:02,786 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c20f09939029475fb9ebd7baaf4fb0d0 as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c20f09939029475fb9ebd7baaf4fb0d0 2024-12-04T20:22:02,791 INFO [M:0;c2ef38372881:38949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c20f09939029475fb9ebd7baaf4fb0d0, entries=1, sequenceid=121, filesize=5.0 K 2024-12-04T20:22:02,793 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd301b3a86c84966b04544984847dda9 as hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bd301b3a86c84966b04544984847dda9 2024-12-04T20:22:02,798 INFO [M:0;c2ef38372881:38949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40221/user/jenkins/test-data/2f9286cf-02f6-0ade-871e-e005e49081a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bd301b3a86c84966b04544984847dda9, entries=1, sequenceid=121, filesize=4.9 K 2024-12-04T20:22:02,799 INFO [M:0;c2ef38372881:38949 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.56 KB/44602, heapSize ~54.88 KB/56192, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=121, compaction requested=false 2024-12-04T20:22:02,801 INFO [M:0;c2ef38372881:38949 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:02,801 DEBUG [M:0;c2ef38372881:38949 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343722678Disabling compacts and flushes for region at 1733343722678Disabling writes for close at 1733343722678Obtaining lock to block concurrent updates at 1733343722678Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733343722678Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44602, getHeapSize=56192, getOffHeapSize=0, getCellsCount=140 at 1733343722679 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733343722679Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733343722679Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733343722694 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733343722694Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733343722704 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733343722717 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733343722717Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733343722727 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733343722739 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733343722739Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733343722749 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733343722762 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733343722762Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c816655: reopening flushed file at 1733343722772 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c0ecebe: reopening flushed file at 1733343722777 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38bdf52d: reopening flushed file at 1733343722785 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70029d5: reopening flushed file at 1733343722792 (+7 ms)Finished flush of dataSize ~43.56 KB/44602, heapSize ~54.88 KB/56192, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=121, compaction requested=false at 1733343722799 (+7 ms)Writing region close event to WAL at 1733343722801 (+2 ms)Closed at 1733343722801 2024-12-04T20:22:02,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:02,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39421 is added to blk_1073741830_1006 (size=52999) 2024-12-04T20:22:02,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741830_1006 (size=52999) 2024-12-04T20:22:02,804 INFO [M:0;c2ef38372881:38949 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:22:02,804 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:22:02,804 INFO [M:0;c2ef38372881:38949 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38949 2024-12-04T20:22:02,804 INFO [M:0;c2ef38372881:38949 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:22:02,906 INFO [M:0;c2ef38372881:38949 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:22:02,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:02,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38949-0x100a6e4cf740000, quorum=127.0.0.1:63107, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:02,910 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@18701e65{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:02,910 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4dfe8781{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:02,910 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:02,911 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39212263{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:02,911 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ec454b0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:02,912 WARN [BP-338094082-172.17.0.2-1733343670495 heartbeating to localhost/127.0.0.1:40221 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:22:02,912 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:22:02,912 WARN [BP-338094082-172.17.0.2-1733343670495 heartbeating to localhost/127.0.0.1:40221 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-338094082-172.17.0.2-1733343670495 (Datanode Uuid a1a53cca-d5a3-4d5c-89ff-ab7d1d772bf2) service to localhost/127.0.0.1:40221 2024-12-04T20:22:02,912 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:22:02,913 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/data/data3/current/BP-338094082-172.17.0.2-1733343670495 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:02,913 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/data/data4/current/BP-338094082-172.17.0.2-1733343670495 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:02,914 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:22:02,916 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6dc3ea71{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:02,916 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5718d675{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:02,917 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:02,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ac0122b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:02,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@67f9152{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:02,918 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:22:02,918 WARN [BP-338094082-172.17.0.2-1733343670495 heartbeating to localhost/127.0.0.1:40221 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:22:02,918 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:22:02,918 WARN [BP-338094082-172.17.0.2-1733343670495 heartbeating to localhost/127.0.0.1:40221 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-338094082-172.17.0.2-1733343670495 (Datanode Uuid 1a6fdab3-4332-4580-b3b6-68fafb5c707e) service to localhost/127.0.0.1:40221 2024-12-04T20:22:02,919 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/data/data1/current/BP-338094082-172.17.0.2-1733343670495 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:02,919 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/cluster_1105e5f4-0abf-7bdd-5fe7-29f6eea66e47/data/data2/current/BP-338094082-172.17.0.2-1733343670495 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:02,919 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:22:02,926 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6bac3ffc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:22:02,927 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4a31a089{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:02,927 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:02,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79c156a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:02,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@294b1089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:02,933 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:22:02,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T20:22:02,958 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=209 (was 182) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40221 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40221 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40221 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c2ef38372881:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40221 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40221 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40221 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40221 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40221 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:40221 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=485 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=119 (was 60) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4379 (was 4464) 2024-12-04T20:22:02,964 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=209, OpenFileDescriptor=485, MaxFileDescriptor=1048576, SystemLoadAverage=119, ProcessCount=11, AvailableMemoryMB=4380 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.log.dir so I do NOT create it in target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0393956d-1005-eb5d-7cdd-3e00ee89591d/hadoop.tmp.dir so I do NOT create it in target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063, deleteOnExit=true 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/test.cache.data in system properties and HBase conf 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:22:02,965 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:22:02,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:22:02,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:22:02,978 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:22:03,015 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:22:03,019 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:22:03,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:22:03,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:22:03,020 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:22:03,020 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:22:03,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a4214d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:22:03,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@110e33ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:22:03,108 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d900229{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/java.io.tmpdir/jetty-localhost-37753-hadoop-hdfs-3_4_1-tests_jar-_-any-4715435793514855749/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:22:03,109 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@62c26df6{HTTP/1.1, (http/1.1)}{localhost:37753} 2024-12-04T20:22:03,109 INFO [Time-limited test {}] server.Server(415): Started @234949ms 2024-12-04T20:22:03,119 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:22:03,200 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:22:03,204 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:22:03,205 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:22:03,205 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:22:03,205 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:22:03,209 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63e2e387{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:22:03,209 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ceb85b6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:22:03,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26d44036{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/java.io.tmpdir/jetty-localhost-39153-hadoop-hdfs-3_4_1-tests_jar-_-any-17245996067105645548/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:03,310 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d8c7847{HTTP/1.1, (http/1.1)}{localhost:39153} 2024-12-04T20:22:03,310 INFO [Time-limited test {}] server.Server(415): Started @235150ms 2024-12-04T20:22:03,311 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:22:03,335 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:22:03,337 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:22:03,338 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:22:03,338 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:22:03,338 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:22:03,338 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@605d7cdf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:22:03,339 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@132e4184{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:22:03,364 WARN [Thread-1969 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/data/data1/current/BP-965759213-172.17.0.2-1733343722982/current, will proceed with Du for space computation calculation, 2024-12-04T20:22:03,364 WARN [Thread-1970 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/data/data2/current/BP-965759213-172.17.0.2-1733343722982/current, will proceed with Du for space computation calculation, 2024-12-04T20:22:03,377 WARN [Thread-1948 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:22:03,379 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6bc22790a2b76438 with lease ID 0x7ad45d55e2f173ea: Processing first storage report for DS-3b83c040-23ac-4802-957c-e9192ca6faca from datanode DatanodeRegistration(127.0.0.1:34413, datanodeUuid=54c70cc2-697f-46c6-bc12-fc6ffbe70312, infoPort=37933, infoSecurePort=0, ipcPort=44145, storageInfo=lv=-57;cid=testClusterID;nsid=1052178909;c=1733343722982) 2024-12-04T20:22:03,380 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6bc22790a2b76438 with lease ID 0x7ad45d55e2f173ea: from storage DS-3b83c040-23ac-4802-957c-e9192ca6faca node DatanodeRegistration(127.0.0.1:34413, datanodeUuid=54c70cc2-697f-46c6-bc12-fc6ffbe70312, infoPort=37933, infoSecurePort=0, ipcPort=44145, storageInfo=lv=-57;cid=testClusterID;nsid=1052178909;c=1733343722982), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:22:03,380 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6bc22790a2b76438 with lease ID 0x7ad45d55e2f173ea: Processing first storage report for DS-4d27565c-7bb4-4d98-b5f5-fc25225d30f2 from datanode DatanodeRegistration(127.0.0.1:34413, datanodeUuid=54c70cc2-697f-46c6-bc12-fc6ffbe70312, infoPort=37933, infoSecurePort=0, ipcPort=44145, storageInfo=lv=-57;cid=testClusterID;nsid=1052178909;c=1733343722982) 2024-12-04T20:22:03,380 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6bc22790a2b76438 with lease ID 0x7ad45d55e2f173ea: from storage DS-4d27565c-7bb4-4d98-b5f5-fc25225d30f2 node DatanodeRegistration(127.0.0.1:34413, datanodeUuid=54c70cc2-697f-46c6-bc12-fc6ffbe70312, infoPort=37933, infoSecurePort=0, ipcPort=44145, storageInfo=lv=-57;cid=testClusterID;nsid=1052178909;c=1733343722982), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:22:03,423 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:22:03,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@191b8d86{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/java.io.tmpdir/jetty-localhost-34283-hadoop-hdfs-3_4_1-tests_jar-_-any-7849701234729734905/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:03,430 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5689196f{HTTP/1.1, (http/1.1)}{localhost:34283} 2024-12-04T20:22:03,430 INFO [Time-limited test {}] server.Server(415): Started @235270ms 2024-12-04T20:22:03,431 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:22:03,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:03,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:03,480 WARN [Thread-1995 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/data/data3/current/BP-965759213-172.17.0.2-1733343722982/current, will proceed with Du for space computation calculation, 2024-12-04T20:22:03,480 WARN [Thread-1996 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/data/data4/current/BP-965759213-172.17.0.2-1733343722982/current, will proceed with Du for space computation calculation, 2024-12-04T20:22:03,495 WARN [Thread-1984 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:22:03,496 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcf050ebc227c17dc with lease ID 0x7ad45d55e2f173eb: Processing first storage report for DS-f4da8c3c-bc6b-45c8-88b9-46cefcfc9444 from datanode DatanodeRegistration(127.0.0.1:35743, datanodeUuid=fc4d7e90-87c0-4126-b320-9db3129140b6, infoPort=43321, infoSecurePort=0, ipcPort=35979, storageInfo=lv=-57;cid=testClusterID;nsid=1052178909;c=1733343722982) 2024-12-04T20:22:03,497 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcf050ebc227c17dc with lease ID 0x7ad45d55e2f173eb: from storage DS-f4da8c3c-bc6b-45c8-88b9-46cefcfc9444 node DatanodeRegistration(127.0.0.1:35743, datanodeUuid=fc4d7e90-87c0-4126-b320-9db3129140b6, infoPort=43321, infoSecurePort=0, ipcPort=35979, storageInfo=lv=-57;cid=testClusterID;nsid=1052178909;c=1733343722982), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:22:03,497 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcf050ebc227c17dc with lease ID 0x7ad45d55e2f173eb: Processing first storage report for DS-de1195a2-0275-4b23-a6f5-93fba21d6666 from datanode DatanodeRegistration(127.0.0.1:35743, datanodeUuid=fc4d7e90-87c0-4126-b320-9db3129140b6, infoPort=43321, infoSecurePort=0, ipcPort=35979, storageInfo=lv=-57;cid=testClusterID;nsid=1052178909;c=1733343722982) 2024-12-04T20:22:03,497 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcf050ebc227c17dc with lease ID 0x7ad45d55e2f173eb: from storage DS-de1195a2-0275-4b23-a6f5-93fba21d6666 node DatanodeRegistration(127.0.0.1:35743, datanodeUuid=fc4d7e90-87c0-4126-b320-9db3129140b6, infoPort=43321, infoSecurePort=0, ipcPort=35979, storageInfo=lv=-57;cid=testClusterID;nsid=1052178909;c=1733343722982), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:22:03,550 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8 2024-12-04T20:22:03,556 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/zookeeper_0, clientPort=60207, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:22:03,558 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60207 2024-12-04T20:22:03,558 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:03,561 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:03,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:22:03,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:22:03,570 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a with version=8 2024-12-04T20:22:03,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase-staging 2024-12-04T20:22:03,572 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:22:03,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:03,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:03,572 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:22:03,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:03,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:22:03,572 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:22:03,572 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:22:03,572 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34749 2024-12-04T20:22:03,573 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34749 connecting to ZooKeeper ensemble=127.0.0.1:60207 2024-12-04T20:22:03,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:347490x0, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:22:03,577 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34749-0x100a6e59c8f0000 connected 2024-12-04T20:22:03,588 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:03,590 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:03,592 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:03,592 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a, hbase.cluster.distributed=false 2024-12-04T20:22:03,594 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:22:03,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34749 2024-12-04T20:22:03,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34749 2024-12-04T20:22:03,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34749 2024-12-04T20:22:03,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34749 2024-12-04T20:22:03,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34749 2024-12-04T20:22:03,612 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:22:03,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:03,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:03,613 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:22:03,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:03,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:22:03,613 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:22:03,613 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:22:03,613 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33877 2024-12-04T20:22:03,614 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33877 connecting to ZooKeeper ensemble=127.0.0.1:60207 2024-12-04T20:22:03,615 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:03,616 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:03,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338770x0, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:22:03,619 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33877-0x100a6e59c8f0001 connected 2024-12-04T20:22:03,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:03,620 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:22:03,620 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:22:03,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:22:03,621 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:22:03,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33877 2024-12-04T20:22:03,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33877 2024-12-04T20:22:03,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33877 2024-12-04T20:22:03,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33877 2024-12-04T20:22:03,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33877 2024-12-04T20:22:03,633 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:34749 2024-12-04T20:22:03,634 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,34749,1733343723571 2024-12-04T20:22:03,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:22:03,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:22:03,635 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,34749,1733343723571 2024-12-04T20:22:03,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:22:03,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:03,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:03,636 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:22:03,637 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,34749,1733343723571 from backup master directory 2024-12-04T20:22:03,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,34749,1733343723571 2024-12-04T20:22:03,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:22:03,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:22:03,637 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:22:03,637 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,34749,1733343723571 2024-12-04T20:22:03,640 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/hbase.id] with ID: 5f3c8fbf-428d-47c5-8c4f-2fd75dae2bba 2024-12-04T20:22:03,640 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/.tmp/hbase.id 2024-12-04T20:22:03,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:22:03,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:22:03,646 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/.tmp/hbase.id]:[hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/hbase.id] 2024-12-04T20:22:03,658 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:03,658 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:22:03,659 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T20:22:03,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:03,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:03,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:22:03,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:22:03,666 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:22:03,667 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:22:03,667 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:22:03,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:22:03,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:22:03,674 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store 2024-12-04T20:22:03,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:22:03,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:22:04,083 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:04,083 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:22:04,083 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:04,083 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:04,083 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:22:04,083 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:04,083 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:04,083 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343724083Disabling compacts and flushes for region at 1733343724083Disabling writes for close at 1733343724083Writing region close event to WAL at 1733343724083Closed at 1733343724083 2024-12-04T20:22:04,085 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/.initializing 2024-12-04T20:22:04,085 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/WALs/c2ef38372881,34749,1733343723571 2024-12-04T20:22:04,090 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C34749%2C1733343723571, suffix=, logDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/WALs/c2ef38372881,34749,1733343723571, archiveDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/oldWALs, maxLogs=10 2024-12-04T20:22:04,091 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C34749%2C1733343723571.1733343724090 2024-12-04T20:22:04,097 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/WALs/c2ef38372881,34749,1733343723571/c2ef38372881%2C34749%2C1733343723571.1733343724090 2024-12-04T20:22:04,098 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43321:43321),(127.0.0.1/127.0.0.1:37933:37933)] 2024-12-04T20:22:04,099 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:22:04,099 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:04,099 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,099 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,103 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:22:04,103 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:22:04,105 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:04,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,108 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:22:04,108 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:04,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:22:04,111 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:04,112 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,113 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,113 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,115 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,115 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,115 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:22:04,117 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:04,119 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:22:04,119 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703686, jitterRate=-0.1052170991897583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:22:04,120 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733343724100Initializing all the Stores at 1733343724100Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343724101 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343724101Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343724101Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343724101Cleaning up temporary data from old regions at 1733343724115 (+14 ms)Region opened successfully at 1733343724120 (+5 ms) 2024-12-04T20:22:04,120 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:22:04,123 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57cbc1c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:22:04,124 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:22:04,125 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:22:04,125 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:22:04,125 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:22:04,125 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T20:22:04,125 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T20:22:04,125 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:22:04,127 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:22:04,128 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:22:04,129 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:22:04,129 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:22:04,130 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:22:04,130 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:22:04,131 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:22:04,132 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:22:04,134 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:22:04,135 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:22:04,136 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:22:04,138 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:22:04,138 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:22:04,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:04,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:04,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:04,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:04,140 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,34749,1733343723571, sessionid=0x100a6e59c8f0000, setting cluster-up flag (Was=false) 2024-12-04T20:22:04,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:04,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:04,143 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:22:04,144 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,34749,1733343723571 2024-12-04T20:22:04,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:04,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:04,148 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:22:04,149 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,34749,1733343723571 2024-12-04T20:22:04,150 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:22:04,151 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:22:04,151 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:22:04,151 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:22:04,152 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,34749,1733343723571 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:22:04,153 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:22:04,153 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:22:04,153 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:22:04,153 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:22:04,153 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:22:04,153 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,153 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:22:04,153 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,155 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:22:04,155 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:22:04,155 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733343754155 2024-12-04T20:22:04,155 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,156 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:22:04,156 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:22:04,156 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:22:04,157 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:22:04,157 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:22:04,157 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343724157,5,FailOnTimeoutGroup] 2024-12-04T20:22:04,157 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343724157,5,FailOnTimeoutGroup] 2024-12-04T20:22:04,157 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,157 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:22:04,157 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,157 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:22:04,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:22:04,162 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:22:04,163 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a 2024-12-04T20:22:04,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:22:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:22:04,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:04,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:22:04,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:22:04,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:22:04,172 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:22:04,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:22:04,173 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:22:04,173 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:22:04,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:22:04,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:22:04,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740 2024-12-04T20:22:04,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740 2024-12-04T20:22:04,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:22:04,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:22:04,176 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:22:04,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:22:04,179 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:22:04,179 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=790656, jitterRate=0.005371168255805969}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:22:04,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733343724168Initializing all the Stores at 1733343724169 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343724169Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343724169Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343724169Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343724169Cleaning up temporary data from old regions at 1733343724176 (+7 ms)Region opened successfully at 1733343724179 (+3 ms) 2024-12-04T20:22:04,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:22:04,180 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:22:04,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:22:04,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:22:04,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:22:04,180 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:22:04,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343724180Disabling compacts and flushes for region at 1733343724180Disabling writes for close at 1733343724180Writing region close event to WAL at 1733343724180Closed at 1733343724180 2024-12-04T20:22:04,181 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:22:04,181 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:22:04,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:22:04,182 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:22:04,183 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:22:04,226 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(746): ClusterId : 5f3c8fbf-428d-47c5-8c4f-2fd75dae2bba 2024-12-04T20:22:04,226 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:22:04,228 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:22:04,228 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:22:04,231 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:22:04,231 DEBUG [RS:0;c2ef38372881:33877 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3883b503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:22:04,244 DEBUG [RS:0;c2ef38372881:33877 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:33877 2024-12-04T20:22:04,244 INFO [RS:0;c2ef38372881:33877 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:22:04,244 INFO [RS:0;c2ef38372881:33877 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:22:04,244 DEBUG [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:22:04,245 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,34749,1733343723571 with port=33877, startcode=1733343723612 2024-12-04T20:22:04,245 DEBUG [RS:0;c2ef38372881:33877 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:22:04,247 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33067, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:22:04,247 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34749 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,247 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34749 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,248 DEBUG [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a 2024-12-04T20:22:04,248 DEBUG [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44207 2024-12-04T20:22:04,248 DEBUG [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:22:04,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:22:04,250 DEBUG [RS:0;c2ef38372881:33877 {}] zookeeper.ZKUtil(111): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,250 WARN [RS:0;c2ef38372881:33877 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:22:04,250 INFO [RS:0;c2ef38372881:33877 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:22:04,250 DEBUG [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,250 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,33877,1733343723612] 2024-12-04T20:22:04,253 INFO [RS:0;c2ef38372881:33877 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:22:04,254 INFO [RS:0;c2ef38372881:33877 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:22:04,254 INFO [RS:0;c2ef38372881:33877 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:22:04,254 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,254 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:22:04,255 INFO [RS:0;c2ef38372881:33877 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:22:04,255 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,255 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,255 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,255 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,255 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,255 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,255 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:22:04,255 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,255 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,256 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,256 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,256 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,256 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:04,256 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:22:04,256 DEBUG [RS:0;c2ef38372881:33877 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:22:04,256 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,256 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,256 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,256 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,256 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,256 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33877,1733343723612-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:22:04,270 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:22:04,270 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,33877,1733343723612-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,270 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,270 INFO [RS:0;c2ef38372881:33877 {}] regionserver.Replication(171): c2ef38372881,33877,1733343723612 started 2024-12-04T20:22:04,281 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,281 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,33877,1733343723612, RpcServer on c2ef38372881/172.17.0.2:33877, sessionid=0x100a6e59c8f0001 2024-12-04T20:22:04,281 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:22:04,281 DEBUG [RS:0;c2ef38372881:33877 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,281 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,33877,1733343723612' 2024-12-04T20:22:04,281 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:22:04,282 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:22:04,283 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:22:04,283 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:22:04,283 DEBUG [RS:0;c2ef38372881:33877 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,283 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,33877,1733343723612' 2024-12-04T20:22:04,283 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:22:04,283 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:22:04,284 DEBUG [RS:0;c2ef38372881:33877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:22:04,284 INFO [RS:0;c2ef38372881:33877 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:22:04,284 INFO [RS:0;c2ef38372881:33877 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:22:04,333 WARN [c2ef38372881:34749 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T20:22:04,387 INFO [RS:0;c2ef38372881:33877 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C33877%2C1733343723612, suffix=, logDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612, archiveDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/oldWALs, maxLogs=32 2024-12-04T20:22:04,388 INFO [RS:0;c2ef38372881:33877 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33877%2C1733343723612.1733343724388 2024-12-04T20:22:04,396 INFO [RS:0;c2ef38372881:33877 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.1733343724388 2024-12-04T20:22:04,402 DEBUG [RS:0;c2ef38372881:33877 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37933:37933),(127.0.0.1/127.0.0.1:43321:43321)] 2024-12-04T20:22:04,460 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:04,460 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:04,584 DEBUG [c2ef38372881:34749 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T20:22:04,584 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,586 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,33877,1733343723612, state=OPENING 2024-12-04T20:22:04,588 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:22:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:04,591 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:22:04,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,33877,1733343723612}] 2024-12-04T20:22:04,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:22:04,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:22:04,746 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:22:04,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45729, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:22:04,753 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:22:04,753 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:22:04,757 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C33877%2C1733343723612.meta, suffix=.meta, logDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612, archiveDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/oldWALs, maxLogs=32 2024-12-04T20:22:04,757 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33877%2C1733343723612.meta.1733343724757.meta 2024-12-04T20:22:04,762 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.meta.1733343724757.meta 2024-12-04T20:22:04,763 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37933:37933),(127.0.0.1/127.0.0.1:43321:43321)] 2024-12-04T20:22:04,763 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:22:04,763 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:22:04,764 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:22:04,764 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:22:04,764 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:22:04,764 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:04,764 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:22:04,764 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:22:04,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:22:04,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:22:04,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:22:04,766 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:22:04,767 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:22:04,767 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:22:04,767 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:22:04,768 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:22:04,768 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:04,769 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:22:04,769 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740 2024-12-04T20:22:04,770 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740 2024-12-04T20:22:04,771 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:22:04,771 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:22:04,771 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:22:04,772 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:22:04,773 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=761217, jitterRate=-0.032063305377960205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:22:04,773 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:22:04,773 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733343724764Writing region info on filesystem at 1733343724764Initializing all the Stores at 1733343724765 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343724765Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343724765Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343724765Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343724765Cleaning up temporary data from old regions at 1733343724771 (+6 ms)Running coprocessor post-open hooks at 1733343724773 (+2 ms)Region opened successfully at 1733343724773 2024-12-04T20:22:04,774 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733343724745 2024-12-04T20:22:04,776 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:22:04,776 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:22:04,777 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,777 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,33877,1733343723612, state=OPEN 2024-12-04T20:22:04,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:22:04,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:22:04,779 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,33877,1733343723612 2024-12-04T20:22:04,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:22:04,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:22:04,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:22:04,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,33877,1733343723612 in 188 msec 2024-12-04T20:22:04,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:22:04,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 601 msec 2024-12-04T20:22:04,785 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:22:04,785 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:22:04,786 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:22:04,786 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,33877,1733343723612, seqNum=-1] 2024-12-04T20:22:04,786 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:22:04,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56631, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:22:04,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 641 msec 2024-12-04T20:22:04,793 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733343724793, completionTime=-1 2024-12-04T20:22:04,793 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T20:22:04,793 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733343784795 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733343844795 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,34749,1733343723571-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,34749,1733343723571-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,34749,1733343723571-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:34749, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,797 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:22:04,800 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.163sec 2024-12-04T20:22:04,800 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:22:04,800 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:22:04,800 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:22:04,800 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:22:04,800 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:22:04,800 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,34749,1733343723571-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:22:04,800 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,34749,1733343723571-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:22:04,803 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:22:04,803 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:22:04,803 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,34749,1733343723571-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:04,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3bfe4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:22:04,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,34749,-1 for getting cluster id 2024-12-04T20:22:04,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:22:04,829 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5f3c8fbf-428d-47c5-8c4f-2fd75dae2bba' 2024-12-04T20:22:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:22:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5f3c8fbf-428d-47c5-8c4f-2fd75dae2bba" 2024-12-04T20:22:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc61e48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:22:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,34749,-1] 2024-12-04T20:22:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:22:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:04,832 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:22:04,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4707bb9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:22:04,833 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:22:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,33877,1733343723612, seqNum=-1] 2024-12-04T20:22:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:22:04,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:22:04,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,34749,1733343723571 2024-12-04T20:22:04,839 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:04,842 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T20:22:04,842 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:22:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,34749,1733343723571 2024-12-04T20:22:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4725c44f 2024-12-04T20:22:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:22:04,845 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:22:04,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34749 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T20:22:04,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T20:22:04,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34749 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:22:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34749 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-12-04T20:22:04,849 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:22:04,849 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:04,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34749 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-12-04T20:22:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:22:04,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:22:04,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741835_1011 (size=381) 2024-12-04T20:22:04,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741835_1011 (size=381) 2024-12-04T20:22:04,860 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1895d4fe5ab48cac297c933e9fe0af62, NAME => 'TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a 2024-12-04T20:22:04,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741836_1012 (size=64) 2024-12-04T20:22:04,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741836_1012 (size=64) 2024-12-04T20:22:04,868 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:04,868 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 1895d4fe5ab48cac297c933e9fe0af62, disabling compactions & flushes 2024-12-04T20:22:04,868 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:04,868 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:04,868 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. after waiting 0 ms 2024-12-04T20:22:04,868 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:04,868 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:04,868 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1895d4fe5ab48cac297c933e9fe0af62: Waiting for close lock at 1733343724868Disabling compacts and flushes for region at 1733343724868Disabling writes for close at 1733343724868Writing region close event to WAL at 1733343724868Closed at 1733343724868 2024-12-04T20:22:04,869 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:22:04,870 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733343724869"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733343724869"}]},"ts":"1733343724869"} 2024-12-04T20:22:04,872 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:22:04,873 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:22:04,873 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343724873"}]},"ts":"1733343724873"} 2024-12-04T20:22:04,874 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-12-04T20:22:04,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, ASSIGN}] 2024-12-04T20:22:04,876 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, ASSIGN 2024-12-04T20:22:04,877 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, ASSIGN; state=OFFLINE, location=c2ef38372881,33877,1733343723612; forceNewPlan=false, retain=false 2024-12-04T20:22:05,028 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1895d4fe5ab48cac297c933e9fe0af62, regionState=OPENING, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:05,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, ASSIGN because future has completed 2024-12-04T20:22:05,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1895d4fe5ab48cac297c933e9fe0af62, server=c2ef38372881,33877,1733343723612}] 2024-12-04T20:22:05,199 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:05,199 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1895d4fe5ab48cac297c933e9fe0af62, NAME => 'TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:22:05,199 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,199 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:05,199 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,199 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,201 INFO [StoreOpener-1895d4fe5ab48cac297c933e9fe0af62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,203 INFO [StoreOpener-1895d4fe5ab48cac297c933e9fe0af62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1895d4fe5ab48cac297c933e9fe0af62 columnFamilyName info 2024-12-04T20:22:05,203 DEBUG [StoreOpener-1895d4fe5ab48cac297c933e9fe0af62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:05,204 INFO [StoreOpener-1895d4fe5ab48cac297c933e9fe0af62-1 {}] regionserver.HStore(327): Store=1895d4fe5ab48cac297c933e9fe0af62/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:05,204 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,205 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,206 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,206 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,206 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,209 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,211 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:22:05,211 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1895d4fe5ab48cac297c933e9fe0af62; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=864027, jitterRate=0.09866778552532196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:22:05,211 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:05,212 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1895d4fe5ab48cac297c933e9fe0af62: Running coprocessor pre-open hook at 1733343725200Writing region info on filesystem at 1733343725200Initializing all the Stores at 1733343725201 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343725201Cleaning up temporary data from old regions at 1733343725206 (+5 ms)Running coprocessor post-open hooks at 1733343725211 (+5 ms)Region opened successfully at 1733343725212 (+1 ms) 2024-12-04T20:22:05,213 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., pid=6, masterSystemTime=1733343725190 2024-12-04T20:22:05,215 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:05,215 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:05,216 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1895d4fe5ab48cac297c933e9fe0af62, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:05,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1895d4fe5ab48cac297c933e9fe0af62, server=c2ef38372881,33877,1733343723612 because future has completed 2024-12-04T20:22:05,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T20:22:05,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1895d4fe5ab48cac297c933e9fe0af62, server=c2ef38372881,33877,1733343723612 in 184 msec 2024-12-04T20:22:05,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T20:22:05,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, ASSIGN in 346 msec 2024-12-04T20:22:05,225 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:22:05,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343725225"}]},"ts":"1733343725225"} 2024-12-04T20:22:05,227 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-12-04T20:22:05,228 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:22:05,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 382 msec 2024-12-04T20:22:05,461 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:05,461 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:06,462 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:06,462 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:07,464 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:07,464 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:07,488 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,488 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,488 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,489 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,489 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,489 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,509 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,510 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,510 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,510 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,510 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,511 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,515 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,515 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,515 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:07,518 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,023 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T20:22:08,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,025 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,025 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,025 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,026 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,026 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,039 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,039 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,039 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,039 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,039 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,040 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,043 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,043 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,043 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,046 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:08,465 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:08,465 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:09,466 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:09,466 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:10,253 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T20:22:10,254 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-12-04T20:22:10,467 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:10,467 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:11,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:11,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:12,469 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:12,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:12,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:22:12,620 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T20:22:12,621 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:22:12,621 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T20:22:12,621 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-12-04T20:22:12,621 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T20:22:13,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:13,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:14,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:14,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:22:14,891 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-12-04T20:22:14,892 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-12-04T20:22:14,895 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-12-04T20:22:14,895 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., hostname=c2ef38372881,33877,1733343723612, seqNum=2] 2024-12-04T20:22:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:14,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:22:14,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/626ee03c74b241898db97bf7b785b578 is 1080, key is row0001/info:/1733343734900/Put/seqid=0 2024-12-04T20:22:14,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741837_1013 (size=12509) 2024-12-04T20:22:14,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741837_1013 (size=12509) 2024-12-04T20:22:14,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/626ee03c74b241898db97bf7b785b578 2024-12-04T20:22:14,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/626ee03c74b241898db97bf7b785b578 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/626ee03c74b241898db97bf7b785b578 2024-12-04T20:22:14,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/626ee03c74b241898db97bf7b785b578, entries=7, sequenceid=11, filesize=12.2 K 2024-12-04T20:22:14,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 1895d4fe5ab48cac297c933e9fe0af62 in 36ms, sequenceid=11, compaction requested=false 2024-12-04T20:22:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:14,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB 2024-12-04T20:22:14,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/53b894df789b4070964e62765d2ade71 is 1080, key is row0008/info:/1733343734914/Put/seqid=0 2024-12-04T20:22:14,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741838_1014 (size=24376) 2024-12-04T20:22:14,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741838_1014 (size=24376) 2024-12-04T20:22:14,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/53b894df789b4070964e62765d2ade71 2024-12-04T20:22:14,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/53b894df789b4070964e62765d2ade71 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71 2024-12-04T20:22:14,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71, entries=18, sequenceid=32, filesize=23.8 K 2024-12-04T20:22:14,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=7.36 KB/7532 for 1895d4fe5ab48cac297c933e9fe0af62 in 22ms, sequenceid=32, compaction requested=false 2024-12-04T20:22:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.0 K, sizeToCheck=16.0 K 2024-12-04T20:22:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71 because midkey is the same as first or last row 2024-12-04T20:22:15,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:15,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:16,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:16,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:16,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB 2024-12-04T20:22:16,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/a998d0df4acd43a0982e80964a62931b is 1080, key is row0026/info:/1733343734952/Put/seqid=0 2024-12-04T20:22:16,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741839_1015 (size=13586) 2024-12-04T20:22:16,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741839_1015 (size=13586) 2024-12-04T20:22:16,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/a998d0df4acd43a0982e80964a62931b 2024-12-04T20:22:17,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/a998d0df4acd43a0982e80964a62931b as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a998d0df4acd43a0982e80964a62931b 2024-12-04T20:22:17,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a998d0df4acd43a0982e80964a62931b, entries=8, sequenceid=43, filesize=13.3 K 2024-12-04T20:22:17,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=16.81 KB/17216 for 1895d4fe5ab48cac297c933e9fe0af62 in 44ms, sequenceid=43, compaction requested=true 2024-12-04T20:22:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71 because midkey is the same as first or last row 2024-12-04T20:22:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1895d4fe5ab48cac297c933e9fe0af62:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:17,012 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:17,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB 2024-12-04T20:22:17,014 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:17,014 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): 1895d4fe5ab48cac297c933e9fe0af62/info is initiating minor compaction (all files) 2024-12-04T20:22:17,014 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1895d4fe5ab48cac297c933e9fe0af62/info in TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:17,014 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/626ee03c74b241898db97bf7b785b578, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a998d0df4acd43a0982e80964a62931b] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp, totalSize=49.3 K 2024-12-04T20:22:17,015 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 626ee03c74b241898db97bf7b785b578, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733343734900 2024-12-04T20:22:17,015 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53b894df789b4070964e62765d2ade71, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1733343734914 2024-12-04T20:22:17,015 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a998d0df4acd43a0982e80964a62931b, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733343734952 2024-12-04T20:22:17,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/b38a27fd8037425c9216551b2d2eaa22 is 1080, key is row0034/info:/1733343736970/Put/seqid=0 2024-12-04T20:22:17,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741840_1016 (size=23299) 2024-12-04T20:22:17,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741840_1016 (size=23299) 2024-12-04T20:22:17,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/b38a27fd8037425c9216551b2d2eaa22 2024-12-04T20:22:17,028 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1895d4fe5ab48cac297c933e9fe0af62#info#compaction#60 average throughput is 33.86 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:17,029 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/9c40355df7474997b9b32cb2d048f4fc is 1080, key is row0001/info:/1733343734900/Put/seqid=0 2024-12-04T20:22:17,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/b38a27fd8037425c9216551b2d2eaa22 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/b38a27fd8037425c9216551b2d2eaa22 2024-12-04T20:22:17,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741841_1017 (size=40670) 2024-12-04T20:22:17,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741841_1017 (size=40670) 2024-12-04T20:22:17,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/b38a27fd8037425c9216551b2d2eaa22, entries=17, sequenceid=63, filesize=22.8 K 2024-12-04T20:22:17,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=10.51 KB/10760 for 1895d4fe5ab48cac297c933e9fe0af62 in 26ms, sequenceid=63, compaction requested=false 2024-12-04T20:22:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.0 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71 because midkey is the same as first or last row 2024-12-04T20:22:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:17,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-04T20:22:17,041 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/9c40355df7474997b9b32cb2d048f4fc as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc 2024-12-04T20:22:17,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/9cd8bcc3d87740e48a7b3da41f7b88fb is 1080, key is row0051/info:/1733343737013/Put/seqid=0 2024-12-04T20:22:17,047 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1895d4fe5ab48cac297c933e9fe0af62/info of 1895d4fe5ab48cac297c933e9fe0af62 into 9c40355df7474997b9b32cb2d048f4fc(size=39.7 K), total size for store is 62.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:17,047 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., storeName=1895d4fe5ab48cac297c933e9fe0af62/info, priority=13, startTime=1733343737012; duration=0sec 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=62.5 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc because midkey is the same as first or last row 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=62.5 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc because midkey is the same as first or last row 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=62.5 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc because midkey is the same as first or last row 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:17,047 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1895d4fe5ab48cac297c933e9fe0af62:info 2024-12-04T20:22:17,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741842_1018 (size=17894) 2024-12-04T20:22:17,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741842_1018 (size=17894) 2024-12-04T20:22:17,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/9cd8bcc3d87740e48a7b3da41f7b88fb 2024-12-04T20:22:17,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/9cd8bcc3d87740e48a7b3da41f7b88fb as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9cd8bcc3d87740e48a7b3da41f7b88fb 2024-12-04T20:22:17,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9cd8bcc3d87740e48a7b3da41f7b88fb, entries=12, sequenceid=78, filesize=17.5 K 2024-12-04T20:22:17,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=2.10 KB/2152 for 1895d4fe5ab48cac297c933e9fe0af62 in 20ms, sequenceid=78, compaction requested=true 2024-12-04T20:22:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=79.9 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc because midkey is the same as first or last row 2024-12-04T20:22:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1895d4fe5ab48cac297c933e9fe0af62:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:17,060 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:17,061 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 81863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:17,062 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): 1895d4fe5ab48cac297c933e9fe0af62/info is initiating minor compaction (all files) 2024-12-04T20:22:17,062 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1895d4fe5ab48cac297c933e9fe0af62/info in TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:17,062 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/b38a27fd8037425c9216551b2d2eaa22, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9cd8bcc3d87740e48a7b3da41f7b88fb] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp, totalSize=79.9 K 2024-12-04T20:22:17,062 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c40355df7474997b9b32cb2d048f4fc, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733343734900 2024-12-04T20:22:17,062 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b38a27fd8037425c9216551b2d2eaa22, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733343736970 2024-12-04T20:22:17,063 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cd8bcc3d87740e48a7b3da41f7b88fb, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733343737013 2024-12-04T20:22:17,074 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1895d4fe5ab48cac297c933e9fe0af62#info#compaction#62 average throughput is 21.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:17,075 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/283e8a19c2214d59aae45ce69c5aa412 is 1080, key is row0001/info:/1733343734900/Put/seqid=0 2024-12-04T20:22:17,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741843_1019 (size=72147) 2024-12-04T20:22:17,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741843_1019 (size=72147) 2024-12-04T20:22:17,085 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/283e8a19c2214d59aae45ce69c5aa412 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 2024-12-04T20:22:17,091 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1895d4fe5ab48cac297c933e9fe0af62/info of 1895d4fe5ab48cac297c933e9fe0af62 into 283e8a19c2214d59aae45ce69c5aa412(size=70.5 K), total size for store is 70.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:17,091 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:17,092 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., storeName=1895d4fe5ab48cac297c933e9fe0af62/info, priority=13, startTime=1733343737060; duration=0sec 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=70.5 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 because midkey is the same as first or last row 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=70.5 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 because midkey is the same as first or last row 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=70.5 K, sizeToCheck=16.0 K 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 because midkey is the same as first or last row 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:17,092 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1895d4fe5ab48cac297c933e9fe0af62:info 2024-12-04T20:22:17,474 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:17,474 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:18,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:18,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:22:19,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/29747d77d42d428e9619c22c331c27ec is 1080, key is row0063/info:/1733343737041/Put/seqid=0 2024-12-04T20:22:19,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741844_1020 (size=12509) 2024-12-04T20:22:19,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741844_1020 (size=12509) 2024-12-04T20:22:19,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/29747d77d42d428e9619c22c331c27ec 2024-12-04T20:22:19,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/29747d77d42d428e9619c22c331c27ec as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29747d77d42d428e9619c22c331c27ec 2024-12-04T20:22:19,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29747d77d42d428e9619c22c331c27ec, entries=7, sequenceid=90, filesize=12.2 K 2024-12-04T20:22:19,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 1895d4fe5ab48cac297c933e9fe0af62 in 27ms, sequenceid=90, compaction requested=false 2024-12-04T20:22:19,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:19,082 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=82.7 K, sizeToCheck=16.0 K 2024-12-04T20:22:19,082 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:19,082 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 because midkey is the same as first or last row 2024-12-04T20:22:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-12-04T20:22:19,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/a8bf562dc79a460bb3b0f7237c0ffbd2 is 1080, key is row0070/info:/1733343739056/Put/seqid=0 2024-12-04T20:22:19,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741845_1021 (size=16817) 2024-12-04T20:22:19,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741845_1021 (size=16817) 2024-12-04T20:22:19,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/a8bf562dc79a460bb3b0f7237c0ffbd2 2024-12-04T20:22:19,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/a8bf562dc79a460bb3b0f7237c0ffbd2 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a8bf562dc79a460bb3b0f7237c0ffbd2 2024-12-04T20:22:19,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a8bf562dc79a460bb3b0f7237c0ffbd2, entries=11, sequenceid=104, filesize=16.4 K 2024-12-04T20:22:19,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for 1895d4fe5ab48cac297c933e9fe0af62 in 28ms, sequenceid=104, compaction requested=true 2024-12-04T20:22:19,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:19,112 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=99.1 K, sizeToCheck=16.0 K 2024-12-04T20:22:19,112 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:19,112 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 because midkey is the same as first or last row 2024-12-04T20:22:19,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1895d4fe5ab48cac297c933e9fe0af62:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:19,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:19,112 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:19,116 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:19,116 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): 1895d4fe5ab48cac297c933e9fe0af62/info is initiating minor compaction (all files) 2024-12-04T20:22:19,116 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1895d4fe5ab48cac297c933e9fe0af62/info in TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:19,116 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29747d77d42d428e9619c22c331c27ec, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a8bf562dc79a460bb3b0f7237c0ffbd2] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp, totalSize=99.1 K 2024-12-04T20:22:19,117 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 283e8a19c2214d59aae45ce69c5aa412, keycount=62, bloomtype=ROW, size=70.5 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733343734900 2024-12-04T20:22:19,118 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29747d77d42d428e9619c22c331c27ec, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733343737041 2024-12-04T20:22:19,118 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8bf562dc79a460bb3b0f7237c0ffbd2, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733343739056 2024-12-04T20:22:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-12-04T20:22:19,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/70a0413710f44bfcb00f5deaa9086a2c is 1080, key is row0081/info:/1733343739086/Put/seqid=0 2024-12-04T20:22:19,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741846_1022 (size=16817) 2024-12-04T20:22:19,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741846_1022 (size=16817) 2024-12-04T20:22:19,146 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1895d4fe5ab48cac297c933e9fe0af62#info#compaction#66 average throughput is 20.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:19,146 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/e39f40ccb6f847a4b881063f1608d56f is 1080, key is row0001/info:/1733343734900/Put/seqid=0 2024-12-04T20:22:19,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/70a0413710f44bfcb00f5deaa9086a2c 2024-12-04T20:22:19,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/70a0413710f44bfcb00f5deaa9086a2c as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/70a0413710f44bfcb00f5deaa9086a2c 2024-12-04T20:22:19,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741847_1023 (size=91639) 2024-12-04T20:22:19,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741847_1023 (size=91639) 2024-12-04T20:22:19,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/70a0413710f44bfcb00f5deaa9086a2c, entries=11, sequenceid=118, filesize=16.4 K 2024-12-04T20:22:19,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=5.25 KB/5380 for 1895d4fe5ab48cac297c933e9fe0af62 in 48ms, sequenceid=118, compaction requested=false 2024-12-04T20:22:19,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:19,169 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/e39f40ccb6f847a4b881063f1608d56f as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f 2024-12-04T20:22:19,169 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=115.5 K, sizeToCheck=16.0 K 2024-12-04T20:22:19,169 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:19,169 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 because midkey is the same as first or last row 2024-12-04T20:22:19,177 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1895d4fe5ab48cac297c933e9fe0af62/info of 1895d4fe5ab48cac297c933e9fe0af62 into e39f40ccb6f847a4b881063f1608d56f(size=89.5 K), total size for store is 105.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:19,177 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1895d4fe5ab48cac297c933e9fe0af62: 2024-12-04T20:22:19,177 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., storeName=1895d4fe5ab48cac297c933e9fe0af62/info, priority=13, startTime=1733343739112; duration=0sec 2024-12-04T20:22:19,177 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K 2024-12-04T20:22:19,177 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:19,177 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K 2024-12-04T20:22:19,177 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:19,177 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K 2024-12-04T20:22:19,177 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T20:22:19,178 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:19,178 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:19,178 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1895d4fe5ab48cac297c933e9fe0af62:info 2024-12-04T20:22:19,180 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34749 {}] assignment.AssignmentManager(1363): Split request from c2ef38372881,33877,1733343723612, parent={ENCODED => 1895d4fe5ab48cac297c933e9fe0af62, NAME => 'TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-12-04T20:22:19,187 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34749 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=c2ef38372881,33877,1733343723612 2024-12-04T20:22:19,192 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34749 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1895d4fe5ab48cac297c933e9fe0af62, daughterA=007562742faf9522bcb062faed9d6eae, daughterB=ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,193 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1895d4fe5ab48cac297c933e9fe0af62, daughterA=007562742faf9522bcb062faed9d6eae, daughterB=ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,193 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1895d4fe5ab48cac297c933e9fe0af62, daughterA=007562742faf9522bcb062faed9d6eae, daughterB=ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,193 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1895d4fe5ab48cac297c933e9fe0af62, daughterA=007562742faf9522bcb062faed9d6eae, daughterB=ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, UNASSIGN}] 2024-12-04T20:22:19,205 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, UNASSIGN 2024-12-04T20:22:19,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1895d4fe5ab48cac297c933e9fe0af62, regionState=CLOSING, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:19,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, UNASSIGN because future has completed 2024-12-04T20:22:19,211 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-12-04T20:22:19,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1895d4fe5ab48cac297c933e9fe0af62, server=c2ef38372881,33877,1733343723612}] 2024-12-04T20:22:19,370 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,370 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-12-04T20:22:19,371 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 1895d4fe5ab48cac297c933e9fe0af62, disabling compactions & flushes 2024-12-04T20:22:19,371 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:19,371 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:19,371 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. after waiting 0 ms 2024-12-04T20:22:19,371 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:19,371 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 1895d4fe5ab48cac297c933e9fe0af62 1/1 column families, dataSize=5.25 KB heapSize=5.88 KB 2024-12-04T20:22:19,375 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/29ea0eabff9b478981ea9313060a54d8 is 1080, key is row0092/info:/1733343739123/Put/seqid=0 2024-12-04T20:22:19,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741848_1024 (size=10347) 2024-12-04T20:22:19,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741848_1024 (size=10347) 2024-12-04T20:22:19,380 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.25 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/29ea0eabff9b478981ea9313060a54d8 2024-12-04T20:22:19,386 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/.tmp/info/29ea0eabff9b478981ea9313060a54d8 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29ea0eabff9b478981ea9313060a54d8 2024-12-04T20:22:19,391 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29ea0eabff9b478981ea9313060a54d8, entries=5, sequenceid=127, filesize=10.1 K 2024-12-04T20:22:19,393 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for 1895d4fe5ab48cac297c933e9fe0af62 in 22ms, sequenceid=127, compaction requested=true 2024-12-04T20:22:19,394 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/626ee03c74b241898db97bf7b785b578, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a998d0df4acd43a0982e80964a62931b, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/b38a27fd8037425c9216551b2d2eaa22, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9cd8bcc3d87740e48a7b3da41f7b88fb, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29747d77d42d428e9619c22c331c27ec, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a8bf562dc79a460bb3b0f7237c0ffbd2] to archive 2024-12-04T20:22:19,395 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:22:19,397 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/626ee03c74b241898db97bf7b785b578 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/626ee03c74b241898db97bf7b785b578 2024-12-04T20:22:19,399 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/53b894df789b4070964e62765d2ade71 2024-12-04T20:22:19,401 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9c40355df7474997b9b32cb2d048f4fc 2024-12-04T20:22:19,402 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a998d0df4acd43a0982e80964a62931b to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a998d0df4acd43a0982e80964a62931b 2024-12-04T20:22:19,404 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/b38a27fd8037425c9216551b2d2eaa22 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/b38a27fd8037425c9216551b2d2eaa22 2024-12-04T20:22:19,405 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/283e8a19c2214d59aae45ce69c5aa412 2024-12-04T20:22:19,406 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9cd8bcc3d87740e48a7b3da41f7b88fb to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/9cd8bcc3d87740e48a7b3da41f7b88fb 2024-12-04T20:22:19,407 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29747d77d42d428e9619c22c331c27ec to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29747d77d42d428e9619c22c331c27ec 2024-12-04T20:22:19,409 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a8bf562dc79a460bb3b0f7237c0ffbd2 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/a8bf562dc79a460bb3b0f7237c0ffbd2 2024-12-04T20:22:19,415 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1 2024-12-04T20:22:19,416 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. 2024-12-04T20:22:19,416 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 1895d4fe5ab48cac297c933e9fe0af62: Waiting for close lock at 1733343739371Running coprocessor pre-close hooks at 1733343739371Disabling compacts and flushes for region at 1733343739371Disabling writes for close at 1733343739371Obtaining lock to block concurrent updates at 1733343739371Preparing flush snapshotting stores in 1895d4fe5ab48cac297c933e9fe0af62 at 1733343739371Finished memstore snapshotting TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., syncing WAL and waiting on mvcc, flushsize=dataSize=5380, getHeapSize=6000, getOffHeapSize=0, getCellsCount=5 at 1733343739371Flushing stores of TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. at 1733343739372 (+1 ms)Flushing 1895d4fe5ab48cac297c933e9fe0af62/info: creating writer at 1733343739372Flushing 1895d4fe5ab48cac297c933e9fe0af62/info: appending metadata at 1733343739375 (+3 ms)Flushing 1895d4fe5ab48cac297c933e9fe0af62/info: closing flushed file at 1733343739375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d0db854: reopening flushed file at 1733343739385 (+10 ms)Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for 1895d4fe5ab48cac297c933e9fe0af62 in 22ms, sequenceid=127, compaction requested=true at 1733343739393 (+8 ms)Writing region close event to WAL at 1733343739411 (+18 ms)Running coprocessor post-close hooks at 1733343739415 (+4 ms)Closed at 1733343739416 (+1 ms) 2024-12-04T20:22:19,418 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,418 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1895d4fe5ab48cac297c933e9fe0af62, regionState=CLOSED 2024-12-04T20:22:19,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1895d4fe5ab48cac297c933e9fe0af62, server=c2ef38372881,33877,1733343723612 because future has completed 2024-12-04T20:22:19,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-04T20:22:19,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 1895d4fe5ab48cac297c933e9fe0af62, server=c2ef38372881,33877,1733343723612 in 209 msec 2024-12-04T20:22:19,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T20:22:19,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1895d4fe5ab48cac297c933e9fe0af62, UNASSIGN in 220 msec 2024-12-04T20:22:19,435 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:19,438 INFO [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=1895d4fe5ab48cac297c933e9fe0af62, threads=3 2024-12-04T20:22:19,439 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29ea0eabff9b478981ea9313060a54d8 for region: 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,441 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/70a0413710f44bfcb00f5deaa9086a2c for region: 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,449 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/70a0413710f44bfcb00f5deaa9086a2c, top=true 2024-12-04T20:22:19,451 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29ea0eabff9b478981ea9313060a54d8, top=true 2024-12-04T20:22:19,455 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f for region: 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,462 INFO [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-70a0413710f44bfcb00f5deaa9086a2c for child: ac5e62e4423d0e893d9b60456ddbebf7, parent: 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,462 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/70a0413710f44bfcb00f5deaa9086a2c for region: 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,463 INFO [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-29ea0eabff9b478981ea9313060a54d8 for child: ac5e62e4423d0e893d9b60456ddbebf7, parent: 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,463 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/29ea0eabff9b478981ea9313060a54d8 for region: 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741849_1025 (size=27) 2024-12-04T20:22:19,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741849_1025 (size=27) 2024-12-04T20:22:19,476 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:19,476 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:19,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741850_1026 (size=27) 2024-12-04T20:22:19,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741850_1026 (size=27) 2024-12-04T20:22:19,488 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f for region: 1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:19,490 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 1895d4fe5ab48cac297c933e9fe0af62 Daughter A: [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62] storefiles, Daughter B: [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-29ea0eabff9b478981ea9313060a54d8, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-70a0413710f44bfcb00f5deaa9086a2c, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62] storefiles. 2024-12-04T20:22:19,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741851_1027 (size=71) 2024-12-04T20:22:19,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741851_1027 (size=71) 2024-12-04T20:22:19,511 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:19,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741852_1028 (size=71) 2024-12-04T20:22:19,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741852_1028 (size=71) 2024-12-04T20:22:19,532 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:19,556 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1 2024-12-04T20:22:19,559 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1 2024-12-04T20:22:19,562 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733343739561"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733343739561"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733343739561"}]},"ts":"1733343739561"} 2024-12-04T20:22:19,562 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733343739561"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733343739561"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733343739561"}]},"ts":"1733343739561"} 2024-12-04T20:22:19,562 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733343739561"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733343739561"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733343739561"}]},"ts":"1733343739561"} 2024-12-04T20:22:19,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=007562742faf9522bcb062faed9d6eae, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ac5e62e4423d0e893d9b60456ddbebf7, ASSIGN}] 2024-12-04T20:22:19,581 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ac5e62e4423d0e893d9b60456ddbebf7, ASSIGN 2024-12-04T20:22:19,581 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=007562742faf9522bcb062faed9d6eae, ASSIGN 2024-12-04T20:22:19,582 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ac5e62e4423d0e893d9b60456ddbebf7, ASSIGN; state=SPLITTING_NEW, location=c2ef38372881,33877,1733343723612; forceNewPlan=false, retain=false 2024-12-04T20:22:19,582 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=007562742faf9522bcb062faed9d6eae, ASSIGN; state=SPLITTING_NEW, location=c2ef38372881,33877,1733343723612; forceNewPlan=false, retain=false 2024-12-04T20:22:19,732 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=007562742faf9522bcb062faed9d6eae, regionState=OPENING, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:19,732 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ac5e62e4423d0e893d9b60456ddbebf7, regionState=OPENING, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:19,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ac5e62e4423d0e893d9b60456ddbebf7, ASSIGN because future has completed 2024-12-04T20:22:19,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac5e62e4423d0e893d9b60456ddbebf7, server=c2ef38372881,33877,1733343723612}] 2024-12-04T20:22:19,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=007562742faf9522bcb062faed9d6eae, ASSIGN because future has completed 2024-12-04T20:22:19,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 007562742faf9522bcb062faed9d6eae, server=c2ef38372881,33877,1733343723612}] 2024-12-04T20:22:19,890 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:19,891 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => ac5e62e4423d0e893d9b60456ddbebf7, NAME => 'TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.', STARTKEY => 'row0062', ENDKEY => ''} 2024-12-04T20:22:19,891 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,891 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:19,891 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,891 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,892 INFO [StoreOpener-ac5e62e4423d0e893d9b60456ddbebf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,893 INFO [StoreOpener-ac5e62e4423d0e893d9b60456ddbebf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac5e62e4423d0e893d9b60456ddbebf7 columnFamilyName info 2024-12-04T20:22:19,893 DEBUG [StoreOpener-ac5e62e4423d0e893d9b60456ddbebf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:19,902 DEBUG [StoreOpener-ac5e62e4423d0e893d9b60456ddbebf7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-29ea0eabff9b478981ea9313060a54d8 2024-12-04T20:22:19,907 DEBUG [StoreOpener-ac5e62e4423d0e893d9b60456ddbebf7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-70a0413710f44bfcb00f5deaa9086a2c 2024-12-04T20:22:19,918 DEBUG [StoreOpener-ac5e62e4423d0e893d9b60456ddbebf7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62->hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f-top 2024-12-04T20:22:19,919 INFO [StoreOpener-ac5e62e4423d0e893d9b60456ddbebf7-1 {}] regionserver.HStore(327): Store=ac5e62e4423d0e893d9b60456ddbebf7/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:19,920 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,921 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,922 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,923 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,923 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,925 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,925 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened ac5e62e4423d0e893d9b60456ddbebf7; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=716800, jitterRate=-0.088542640209198}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:22:19,925 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:19,926 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for ac5e62e4423d0e893d9b60456ddbebf7: Running coprocessor pre-open hook at 1733343739891Writing region info on filesystem at 1733343739891Initializing all the Stores at 1733343739892 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343739892Cleaning up temporary data from old regions at 1733343739923 (+31 ms)Running coprocessor post-open hooks at 1733343739925 (+2 ms)Region opened successfully at 1733343739926 (+1 ms) 2024-12-04T20:22:19,927 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., pid=12, masterSystemTime=1733343739887 2024-12-04T20:22:19,928 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:19,928 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store ac5e62e4423d0e893d9b60456ddbebf7:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:19,928 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:19,930 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:19,930 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:19,930 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:19,931 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 007562742faf9522bcb062faed9d6eae, NAME => 'TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae.', STARTKEY => '', ENDKEY => 'row0062'} 2024-12-04T20:22:19,931 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,931 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:19,931 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,931 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,931 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ac5e62e4423d0e893d9b60456ddbebf7, regionState=OPEN, openSeqNum=131, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:19,933 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:19,933 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): ac5e62e4423d0e893d9b60456ddbebf7/info is initiating minor compaction (all files) 2024-12-04T20:22:19,933 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac5e62e4423d0e893d9b60456ddbebf7/info in TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:19,933 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62->hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f-top, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-70a0413710f44bfcb00f5deaa9086a2c, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-29ea0eabff9b478981ea9313060a54d8] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp, totalSize=116.0 K 2024-12-04T20:22:19,933 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-12-04T20:22:19,934 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T20:22:19,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB 2024-12-04T20:22:19,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac5e62e4423d0e893d9b60456ddbebf7, server=c2ef38372881,33877,1733343723612 because future has completed 2024-12-04T20:22:19,935 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62, keycount=40, bloomtype=ROW, size=89.5 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733343734900 2024-12-04T20:22:19,936 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-70a0413710f44bfcb00f5deaa9086a2c, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733343739086 2024-12-04T20:22:19,936 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-29ea0eabff9b478981ea9313060a54d8, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733343739123 2024-12-04T20:22:19,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T20:22:19,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure ac5e62e4423d0e893d9b60456ddbebf7, server=c2ef38372881,33877,1733343723612 in 201 msec 2024-12-04T20:22:19,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ac5e62e4423d0e893d9b60456ddbebf7, ASSIGN in 362 msec 2024-12-04T20:22:19,944 INFO [StoreOpener-007562742faf9522bcb062faed9d6eae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,947 INFO [StoreOpener-007562742faf9522bcb062faed9d6eae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 007562742faf9522bcb062faed9d6eae columnFamilyName info 2024-12-04T20:22:19,947 DEBUG [StoreOpener-007562742faf9522bcb062faed9d6eae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:19,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/info/6f08be4b6a5c4dd588e0e2c0e0d80cb2 is 193, key is TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7./info:regioninfo/1733343739931/Put/seqid=0 2024-12-04T20:22:19,972 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac5e62e4423d0e893d9b60456ddbebf7#info#compaction#69 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:19,972 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/fce955b8d0454f7f9962c2cf5a00ba1c is 1080, key is row0062/info:/1733343737039/Put/seqid=0 2024-12-04T20:22:19,975 DEBUG [StoreOpener-007562742faf9522bcb062faed9d6eae-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62->hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f-bottom 2024-12-04T20:22:19,975 INFO [StoreOpener-007562742faf9522bcb062faed9d6eae-1 {}] regionserver.HStore(327): Store=007562742faf9522bcb062faed9d6eae/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:19,975 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,976 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,977 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,978 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,978 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,980 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,981 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 007562742faf9522bcb062faed9d6eae; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=782565, jitterRate=-0.0049172937870025635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:22:19,981 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:19,981 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 007562742faf9522bcb062faed9d6eae: Running coprocessor pre-open hook at 1733343739931Writing region info on filesystem at 1733343739931Initializing all the Stores at 1733343739934 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343739934Cleaning up temporary data from old regions at 1733343739978 (+44 ms)Running coprocessor post-open hooks at 1733343739981 (+3 ms)Region opened successfully at 1733343739981 2024-12-04T20:22:19,982 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae., pid=13, masterSystemTime=1733343739887 2024-12-04T20:22:19,982 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 007562742faf9522bcb062faed9d6eae:info, priority=-2147483648, current under compaction store size is 2 2024-12-04T20:22:19,982 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:19,982 DEBUG [RS:0;c2ef38372881:33877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-04T20:22:19,983 INFO [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:19,983 DEBUG [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.HStore(1541): 007562742faf9522bcb062faed9d6eae/info is initiating minor compaction (all files) 2024-12-04T20:22:19,983 INFO [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 007562742faf9522bcb062faed9d6eae/info in TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:19,983 INFO [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62->hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f-bottom] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/.tmp, totalSize=89.5 K 2024-12-04T20:22:19,983 DEBUG [RS:0;c2ef38372881:33877-longCompactions-0 {}] compactions.Compactor(225): Compacting e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62, keycount=40, bloomtype=ROW, size=89.5 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733343734900 2024-12-04T20:22:19,984 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:19,984 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:19,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=007562742faf9522bcb062faed9d6eae, regionState=OPEN, openSeqNum=131, regionLocation=c2ef38372881,33877,1733343723612 2024-12-04T20:22:19,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 007562742faf9522bcb062faed9d6eae, server=c2ef38372881,33877,1733343723612 because future has completed 2024-12-04T20:22:19,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741853_1029 (size=9882) 2024-12-04T20:22:19,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741853_1029 (size=9882) 2024-12-04T20:22:19,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10 2024-12-04T20:22:19,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 007562742faf9522bcb062faed9d6eae, server=c2ef38372881,33877,1733343723612 in 255 msec 2024-12-04T20:22:19,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/info/6f08be4b6a5c4dd588e0e2c0e0d80cb2 2024-12-04T20:22:19,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-12-04T20:22:19,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=007562742faf9522bcb062faed9d6eae, ASSIGN in 414 msec 2024-12-04T20:22:19,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1895d4fe5ab48cac297c933e9fe0af62, daughterA=007562742faf9522bcb062faed9d6eae, daughterB=ac5e62e4423d0e893d9b60456ddbebf7 in 808 msec 2024-12-04T20:22:20,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741854_1030 (size=42984) 2024-12-04T20:22:20,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741854_1030 (size=42984) 2024-12-04T20:22:20,013 INFO [RS:0;c2ef38372881:33877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 007562742faf9522bcb062faed9d6eae#info#compaction#70 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:20,014 DEBUG [RS:0;c2ef38372881:33877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/.tmp/info/bed881ed821e4a959909428a781a7d2d is 1080, key is row0001/info:/1733343734900/Put/seqid=0 2024-12-04T20:22:20,021 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/fce955b8d0454f7f9962c2cf5a00ba1c as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fce955b8d0454f7f9962c2cf5a00ba1c 2024-12-04T20:22:20,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741855_1031 (size=70862) 2024-12-04T20:22:20,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741855_1031 (size=70862) 2024-12-04T20:22:20,032 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac5e62e4423d0e893d9b60456ddbebf7/info of ac5e62e4423d0e893d9b60456ddbebf7 into fce955b8d0454f7f9962c2cf5a00ba1c(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:20,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/ns/7a0fb5a1ff39494a8a7888940bacdefb is 43, key is default/ns:d/1733343724788/Put/seqid=0 2024-12-04T20:22:20,032 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:20,032 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., storeName=ac5e62e4423d0e893d9b60456ddbebf7/info, priority=13, startTime=1733343739927; duration=0sec 2024-12-04T20:22:20,032 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:20,032 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac5e62e4423d0e893d9b60456ddbebf7:info 2024-12-04T20:22:20,033 DEBUG [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/.tmp/info/bed881ed821e4a959909428a781a7d2d as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/info/bed881ed821e4a959909428a781a7d2d 2024-12-04T20:22:20,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741856_1032 (size=5153) 2024-12-04T20:22:20,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741856_1032 (size=5153) 2024-12-04T20:22:20,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/ns/7a0fb5a1ff39494a8a7888940bacdefb 2024-12-04T20:22:20,040 INFO [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 007562742faf9522bcb062faed9d6eae/info of 007562742faf9522bcb062faed9d6eae into bed881ed821e4a959909428a781a7d2d(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:20,040 DEBUG [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 007562742faf9522bcb062faed9d6eae: 2024-12-04T20:22:20,040 INFO [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae., storeName=007562742faf9522bcb062faed9d6eae/info, priority=15, startTime=1733343739982; duration=0sec 2024-12-04T20:22:20,040 DEBUG [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:20,040 DEBUG [RS:0;c2ef38372881:33877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 007562742faf9522bcb062faed9d6eae:info 2024-12-04T20:22:20,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/table/56553b4b51734010bb325be89f5825d9 is 65, key is TestLogRolling-testLogRolling/table:state/1733343725225/Put/seqid=0 2024-12-04T20:22:20,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741857_1033 (size=5340) 2024-12-04T20:22:20,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741857_1033 (size=5340) 2024-12-04T20:22:20,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/table/56553b4b51734010bb325be89f5825d9 2024-12-04T20:22:20,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/info/6f08be4b6a5c4dd588e0e2c0e0d80cb2 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/info/6f08be4b6a5c4dd588e0e2c0e0d80cb2 2024-12-04T20:22:20,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/info/6f08be4b6a5c4dd588e0e2c0e0d80cb2, entries=30, sequenceid=17, filesize=9.7 K 2024-12-04T20:22:20,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/ns/7a0fb5a1ff39494a8a7888940bacdefb as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/ns/7a0fb5a1ff39494a8a7888940bacdefb 2024-12-04T20:22:20,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/ns/7a0fb5a1ff39494a8a7888940bacdefb, entries=2, sequenceid=17, filesize=5.0 K 2024-12-04T20:22:20,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/table/56553b4b51734010bb325be89f5825d9 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/table/56553b4b51734010bb325be89f5825d9 2024-12-04T20:22:20,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/table/56553b4b51734010bb325be89f5825d9, entries=2, sequenceid=17, filesize=5.2 K 2024-12-04T20:22:20,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 164ms, sequenceid=17, compaction requested=false 2024-12-04T20:22:20,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-04T20:22:20,476 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:20,476 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:45948 deadline: 1733343751138, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. is not online on c2ef38372881,33877,1733343723612 2024-12-04T20:22:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., hostname=c2ef38372881,33877,1733343723612, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., hostname=c2ef38372881,33877,1733343723612, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. is not online on c2ef38372881,33877,1733343723612 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:22:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., hostname=c2ef38372881,33877,1733343723612, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62. is not online on c2ef38372881,33877,1733343723612 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T20:22:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733343724845.1895d4fe5ab48cac297c933e9fe0af62., hostname=c2ef38372881,33877,1733343723612, seqNum=2 from cache 2024-12-04T20:22:21,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:21,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:22,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:22,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:23,478 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:23,478 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:24,416 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,416 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,417 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,417 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,417 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,417 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,418 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,419 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,434 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,434 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,434 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,435 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,435 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,435 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,438 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,438 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,438 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,440 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,478 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:24,478 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:24,947 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T20:22:24,948 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,949 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,949 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,949 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,950 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,950 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,971 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,972 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,972 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,972 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,972 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,973 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,975 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,976 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,976 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:24,978 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T20:22:25,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:25,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:26,480 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:26,480 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:27,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:27,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:28,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:28,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:29,482 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:29,482 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:30,483 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:30,483 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., hostname=c2ef38372881,33877,1733343723612, seqNum=131] 2024-12-04T20:22:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:31,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:22:31,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/1573df7e20b946b193c4e138c88d3d9f is 1080, key is row0097/info:/1733343751185/Put/seqid=0 2024-12-04T20:22:31,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741858_1034 (size=12516) 2024-12-04T20:22:31,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741858_1034 (size=12516) 2024-12-04T20:22:31,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/1573df7e20b946b193c4e138c88d3d9f 2024-12-04T20:22:31,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/1573df7e20b946b193c4e138c88d3d9f as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1573df7e20b946b193c4e138c88d3d9f 2024-12-04T20:22:31,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1573df7e20b946b193c4e138c88d3d9f, entries=7, sequenceid=141, filesize=12.2 K 2024-12-04T20:22:31,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for ac5e62e4423d0e893d9b60456ddbebf7 in 26ms, sequenceid=141, compaction requested=false 2024-12-04T20:22:31,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:31,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-04T20:22:31,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/686aad72a6414639ab14fb0033fe4983 is 1080, key is row0104/info:/1733343751198/Put/seqid=0 2024-12-04T20:22:31,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741859_1035 (size=19000) 2024-12-04T20:22:31,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741859_1035 (size=19000) 2024-12-04T20:22:31,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/686aad72a6414639ab14fb0033fe4983 2024-12-04T20:22:31,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/686aad72a6414639ab14fb0033fe4983 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/686aad72a6414639ab14fb0033fe4983 2024-12-04T20:22:31,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/686aad72a6414639ab14fb0033fe4983, entries=13, sequenceid=157, filesize=18.6 K 2024-12-04T20:22:31,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=11.56 KB/11836 for ac5e62e4423d0e893d9b60456ddbebf7 in 23ms, sequenceid=157, compaction requested=true 2024-12-04T20:22:31,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:31,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac5e62e4423d0e893d9b60456ddbebf7:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:31,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:31,248 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:31,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-04T20:22:31,250 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 74500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:31,250 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): ac5e62e4423d0e893d9b60456ddbebf7/info is initiating minor compaction (all files) 2024-12-04T20:22:31,250 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac5e62e4423d0e893d9b60456ddbebf7/info in TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:31,250 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fce955b8d0454f7f9962c2cf5a00ba1c, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1573df7e20b946b193c4e138c88d3d9f, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/686aad72a6414639ab14fb0033fe4983] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp, totalSize=72.8 K 2024-12-04T20:22:31,250 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce955b8d0454f7f9962c2cf5a00ba1c, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733343737039 2024-12-04T20:22:31,251 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1573df7e20b946b193c4e138c88d3d9f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733343751185 2024-12-04T20:22:31,251 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 686aad72a6414639ab14fb0033fe4983, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733343751198 2024-12-04T20:22:31,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/07885ee874e54d10bec8f7bb7962ed31 is 1080, key is row0117/info:/1733343751226/Put/seqid=0 2024-12-04T20:22:31,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741860_1036 (size=17906) 2024-12-04T20:22:31,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741860_1036 (size=17906) 2024-12-04T20:22:31,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/07885ee874e54d10bec8f7bb7962ed31 2024-12-04T20:22:31,261 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac5e62e4423d0e893d9b60456ddbebf7#info#compaction#76 average throughput is 56.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:31,262 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/0df0f13920614451ae68ec93382039bd is 1080, key is row0062/info:/1733343737039/Put/seqid=0 2024-12-04T20:22:31,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/07885ee874e54d10bec8f7bb7962ed31 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07885ee874e54d10bec8f7bb7962ed31 2024-12-04T20:22:31,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741861_1037 (size=64714) 2024-12-04T20:22:31,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741861_1037 (size=64714) 2024-12-04T20:22:31,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07885ee874e54d10bec8f7bb7962ed31, entries=12, sequenceid=172, filesize=17.5 K 2024-12-04T20:22:31,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for ac5e62e4423d0e893d9b60456ddbebf7 in 19ms, sequenceid=172, compaction requested=false 2024-12-04T20:22:31,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:31,271 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/0df0f13920614451ae68ec93382039bd as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0df0f13920614451ae68ec93382039bd 2024-12-04T20:22:31,277 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac5e62e4423d0e893d9b60456ddbebf7/info of ac5e62e4423d0e893d9b60456ddbebf7 into 0df0f13920614451ae68ec93382039bd(size=63.2 K), total size for store is 80.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:31,277 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:31,277 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., storeName=ac5e62e4423d0e893d9b60456ddbebf7/info, priority=13, startTime=1733343751248; duration=0sec 2024-12-04T20:22:31,277 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:31,277 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac5e62e4423d0e893d9b60456ddbebf7:info 2024-12-04T20:22:31,484 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:31,484 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:32,486 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:32,486 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:33,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:33,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:22:33,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/5e7fcf102bb5468f85b87c3cff5851d4 is 1080, key is row0129/info:/1733343753252/Put/seqid=0 2024-12-04T20:22:33,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741862_1038 (size=12516) 2024-12-04T20:22:33,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741862_1038 (size=12516) 2024-12-04T20:22:33,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/5e7fcf102bb5468f85b87c3cff5851d4 2024-12-04T20:22:33,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/5e7fcf102bb5468f85b87c3cff5851d4 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/5e7fcf102bb5468f85b87c3cff5851d4 2024-12-04T20:22:33,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/5e7fcf102bb5468f85b87c3cff5851d4, entries=7, sequenceid=183, filesize=12.2 K 2024-12-04T20:22:33,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for ac5e62e4423d0e893d9b60456ddbebf7 in 26ms, sequenceid=183, compaction requested=true 2024-12-04T20:22:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac5e62e4423d0e893d9b60456ddbebf7:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:33,298 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:33,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-04T20:22:33,299 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 95136 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:33,299 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): ac5e62e4423d0e893d9b60456ddbebf7/info is initiating minor compaction (all files) 2024-12-04T20:22:33,299 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac5e62e4423d0e893d9b60456ddbebf7/info in TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:33,299 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0df0f13920614451ae68ec93382039bd, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07885ee874e54d10bec8f7bb7962ed31, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/5e7fcf102bb5468f85b87c3cff5851d4] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp, totalSize=92.9 K 2024-12-04T20:22:33,300 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df0f13920614451ae68ec93382039bd, keycount=55, bloomtype=ROW, size=63.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733343737039 2024-12-04T20:22:33,300 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07885ee874e54d10bec8f7bb7962ed31, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733343751226 2024-12-04T20:22:33,301 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e7fcf102bb5468f85b87c3cff5851d4, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733343753252 2024-12-04T20:22:33,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/3731cd62ab3c4d9e873fd9a18b5368ff is 1080, key is row0136/info:/1733343753272/Put/seqid=0 2024-12-04T20:22:33,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741863_1039 (size=19000) 2024-12-04T20:22:33,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741863_1039 (size=19000) 2024-12-04T20:22:33,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/3731cd62ab3c4d9e873fd9a18b5368ff 2024-12-04T20:22:33,315 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac5e62e4423d0e893d9b60456ddbebf7#info#compaction#79 average throughput is 37.97 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:33,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/3731cd62ab3c4d9e873fd9a18b5368ff as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/3731cd62ab3c4d9e873fd9a18b5368ff 2024-12-04T20:22:33,315 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/4ece84ce22cc485a8cd187267045cf1d is 1080, key is row0062/info:/1733343737039/Put/seqid=0 2024-12-04T20:22:33,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741864_1040 (size=85371) 2024-12-04T20:22:33,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741864_1040 (size=85371) 2024-12-04T20:22:33,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/3731cd62ab3c4d9e873fd9a18b5368ff, entries=13, sequenceid=199, filesize=18.6 K 2024-12-04T20:22:33,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=9.46 KB/9684 for ac5e62e4423d0e893d9b60456ddbebf7 in 25ms, sequenceid=199, compaction requested=false 2024-12-04T20:22:33,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:33,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB 2024-12-04T20:22:33,327 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/4ece84ce22cc485a8cd187267045cf1d as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/4ece84ce22cc485a8cd187267045cf1d 2024-12-04T20:22:33,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/fa550ae2a9824d32a4da4d4dfdb7cc55 is 1080, key is row0149/info:/1733343753300/Put/seqid=0 2024-12-04T20:22:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741865_1041 (size=15750) 2024-12-04T20:22:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741865_1041 (size=15750) 2024-12-04T20:22:33,334 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac5e62e4423d0e893d9b60456ddbebf7/info of ac5e62e4423d0e893d9b60456ddbebf7 into 4ece84ce22cc485a8cd187267045cf1d(size=83.4 K), total size for store is 101.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:33,334 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:33,334 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., storeName=ac5e62e4423d0e893d9b60456ddbebf7/info, priority=13, startTime=1733343753298; duration=0sec 2024-12-04T20:22:33,334 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:33,334 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac5e62e4423d0e893d9b60456ddbebf7:info 2024-12-04T20:22:33,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/fa550ae2a9824d32a4da4d4dfdb7cc55 2024-12-04T20:22:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/fa550ae2a9824d32a4da4d4dfdb7cc55 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fa550ae2a9824d32a4da4d4dfdb7cc55 2024-12-04T20:22:33,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fa550ae2a9824d32a4da4d4dfdb7cc55, entries=10, sequenceid=212, filesize=15.4 K 2024-12-04T20:22:33,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2152 for ac5e62e4423d0e893d9b60456ddbebf7 in 20ms, sequenceid=212, compaction requested=true 2024-12-04T20:22:33,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:33,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac5e62e4423d0e893d9b60456ddbebf7:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:33,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:33,345 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:33,346 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120121 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:33,346 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): ac5e62e4423d0e893d9b60456ddbebf7/info is initiating minor compaction (all files) 2024-12-04T20:22:33,346 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac5e62e4423d0e893d9b60456ddbebf7/info in TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:33,346 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/4ece84ce22cc485a8cd187267045cf1d, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/3731cd62ab3c4d9e873fd9a18b5368ff, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fa550ae2a9824d32a4da4d4dfdb7cc55] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp, totalSize=117.3 K 2024-12-04T20:22:33,347 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ece84ce22cc485a8cd187267045cf1d, keycount=74, bloomtype=ROW, size=83.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733343737039 2024-12-04T20:22:33,347 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3731cd62ab3c4d9e873fd9a18b5368ff, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733343753272 2024-12-04T20:22:33,347 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa550ae2a9824d32a4da4d4dfdb7cc55, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733343753300 2024-12-04T20:22:33,358 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac5e62e4423d0e893d9b60456ddbebf7#info#compaction#81 average throughput is 49.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:33,358 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/a2be8683b66749a79470d2a5d5efc9aa is 1080, key is row0062/info:/1733343737039/Put/seqid=0 2024-12-04T20:22:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741866_1042 (size=110275) 2024-12-04T20:22:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741866_1042 (size=110275) 2024-12-04T20:22:33,368 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/a2be8683b66749a79470d2a5d5efc9aa as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a2be8683b66749a79470d2a5d5efc9aa 2024-12-04T20:22:33,374 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac5e62e4423d0e893d9b60456ddbebf7/info of ac5e62e4423d0e893d9b60456ddbebf7 into a2be8683b66749a79470d2a5d5efc9aa(size=107.7 K), total size for store is 107.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:33,374 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:33,374 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., storeName=ac5e62e4423d0e893d9b60456ddbebf7/info, priority=13, startTime=1733343753345; duration=0sec 2024-12-04T20:22:33,374 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:33,374 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac5e62e4423d0e893d9b60456ddbebf7:info 2024-12-04T20:22:33,486 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:33,486 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:33,549 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:22:34,487 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:34,487 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:35,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:22:35,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/35edc55385324eaf8f2eb23ffbca3841 is 1080, key is row0159/info:/1733343753327/Put/seqid=0 2024-12-04T20:22:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741867_1043 (size=12516) 2024-12-04T20:22:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741867_1043 (size=12516) 2024-12-04T20:22:35,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/35edc55385324eaf8f2eb23ffbca3841 2024-12-04T20:22:35,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/35edc55385324eaf8f2eb23ffbca3841 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/35edc55385324eaf8f2eb23ffbca3841 2024-12-04T20:22:35,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/35edc55385324eaf8f2eb23ffbca3841, entries=7, sequenceid=224, filesize=12.2 K 2024-12-04T20:22:35,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for ac5e62e4423d0e893d9b60456ddbebf7 in 25ms, sequenceid=224, compaction requested=false 2024-12-04T20:22:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:35,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-04T20:22:35,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/15d2ef595e534f3888c5c86e295622da is 1080, key is row0166/info:/1733343755343/Put/seqid=0 2024-12-04T20:22:35,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741868_1044 (size=19000) 2024-12-04T20:22:35,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741868_1044 (size=19000) 2024-12-04T20:22:35,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/15d2ef595e534f3888c5c86e295622da 2024-12-04T20:22:35,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/15d2ef595e534f3888c5c86e295622da as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/15d2ef595e534f3888c5c86e295622da 2024-12-04T20:22:35,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/15d2ef595e534f3888c5c86e295622da, entries=13, sequenceid=240, filesize=18.6 K 2024-12-04T20:22:35,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=14.71 KB/15064 for ac5e62e4423d0e893d9b60456ddbebf7 in 48ms, sequenceid=240, compaction requested=true 2024-12-04T20:22:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac5e62e4423d0e893d9b60456ddbebf7:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:35,420 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:35,422 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 141791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:35,422 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): ac5e62e4423d0e893d9b60456ddbebf7/info is initiating minor compaction (all files) 2024-12-04T20:22:35,422 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac5e62e4423d0e893d9b60456ddbebf7/info in TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:35,422 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a2be8683b66749a79470d2a5d5efc9aa, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/35edc55385324eaf8f2eb23ffbca3841, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/15d2ef595e534f3888c5c86e295622da] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp, totalSize=138.5 K 2024-12-04T20:22:35,423 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2be8683b66749a79470d2a5d5efc9aa, keycount=97, bloomtype=ROW, size=107.7 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733343737039 2024-12-04T20:22:35,424 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35edc55385324eaf8f2eb23ffbca3841, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733343753327 2024-12-04T20:22:35,424 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d2ef595e534f3888c5c86e295622da, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733343755343 2024-12-04T20:22:35,442 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac5e62e4423d0e893d9b60456ddbebf7#info#compaction#84 average throughput is 60.03 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:35,442 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/8f832ccb477d4c938ec57343bee375af is 1080, key is row0062/info:/1733343737039/Put/seqid=0 2024-12-04T20:22:35,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741869_1045 (size=132069) 2024-12-04T20:22:35,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741869_1045 (size=132069) 2024-12-04T20:22:35,471 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/8f832ccb477d4c938ec57343bee375af as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8f832ccb477d4c938ec57343bee375af 2024-12-04T20:22:35,477 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac5e62e4423d0e893d9b60456ddbebf7/info of ac5e62e4423d0e893d9b60456ddbebf7 into 8f832ccb477d4c938ec57343bee375af(size=129.0 K), total size for store is 129.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:35,477 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:35,477 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., storeName=ac5e62e4423d0e893d9b60456ddbebf7/info, priority=13, startTime=1733343755420; duration=0sec 2024-12-04T20:22:35,477 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:35,477 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac5e62e4423d0e893d9b60456ddbebf7:info 2024-12-04T20:22:35,488 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:35,488 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:36,488 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:36,488 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:37,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-12-04T20:22:37,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/1adf88a92b224b73ac36e83383c2c679 is 1080, key is row0179/info:/1733343755373/Put/seqid=0 2024-12-04T20:22:37,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741870_1046 (size=21157) 2024-12-04T20:22:37,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741870_1046 (size=21157) 2024-12-04T20:22:37,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/1adf88a92b224b73ac36e83383c2c679 2024-12-04T20:22:37,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/1adf88a92b224b73ac36e83383c2c679 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1adf88a92b224b73ac36e83383c2c679 2024-12-04T20:22:37,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1adf88a92b224b73ac36e83383c2c679, entries=15, sequenceid=259, filesize=20.7 K 2024-12-04T20:22:37,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=14.71 KB/15064 for ac5e62e4423d0e893d9b60456ddbebf7 in 31ms, sequenceid=259, compaction requested=false 2024-12-04T20:22:37,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:37,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-12-04T20:22:37,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/a6c41495b3964e4b8c7cb31d84d71332 is 1080, key is row0194/info:/1733343757419/Put/seqid=0 2024-12-04T20:22:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741871_1047 (size=21171) 2024-12-04T20:22:37,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741871_1047 (size=21171) 2024-12-04T20:22:37,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/a6c41495b3964e4b8c7cb31d84d71332 2024-12-04T20:22:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/a6c41495b3964e4b8c7cb31d84d71332 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a6c41495b3964e4b8c7cb31d84d71332 2024-12-04T20:22:37,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a6c41495b3964e4b8c7cb31d84d71332, entries=15, sequenceid=277, filesize=20.7 K 2024-12-04T20:22:37,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=10.51 KB/10760 for ac5e62e4423d0e893d9b60456ddbebf7 in 20ms, sequenceid=277, compaction requested=true 2024-12-04T20:22:37,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:37,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac5e62e4423d0e893d9b60456ddbebf7:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:37,469 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:37,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:37,470 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 174397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:37,470 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): ac5e62e4423d0e893d9b60456ddbebf7/info is initiating minor compaction (all files) 2024-12-04T20:22:37,470 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac5e62e4423d0e893d9b60456ddbebf7/info in TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:37,470 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8f832ccb477d4c938ec57343bee375af, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1adf88a92b224b73ac36e83383c2c679, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a6c41495b3964e4b8c7cb31d84d71332] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp, totalSize=170.3 K 2024-12-04T20:22:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:37,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-04T20:22:37,471 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f832ccb477d4c938ec57343bee375af, keycount=117, bloomtype=ROW, size=129.0 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733343737039 2024-12-04T20:22:37,471 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1adf88a92b224b73ac36e83383c2c679, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733343755373 2024-12-04T20:22:37,471 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c41495b3964e4b8c7cb31d84d71332, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733343757419 2024-12-04T20:22:37,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/f4bc00690d434993a22f0802ac958439 is 1080, key is row0209/info:/1733343757449/Put/seqid=0 2024-12-04T20:22:37,484 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac5e62e4423d0e893d9b60456ddbebf7#info#compaction#88 average throughput is 50.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:37,485 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/42d02823b91a4131b97e5c2d827291ee is 1080, key is row0062/info:/1733343737039/Put/seqid=0 2024-12-04T20:22:37,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741872_1048 (size=17918) 2024-12-04T20:22:37,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741872_1048 (size=17918) 2024-12-04T20:22:37,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741873_1049 (size=164604) 2024-12-04T20:22:37,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741873_1049 (size=164604) 2024-12-04T20:22:37,489 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:37,489 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:37,494 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/42d02823b91a4131b97e5c2d827291ee as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/42d02823b91a4131b97e5c2d827291ee 2024-12-04T20:22:37,500 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac5e62e4423d0e893d9b60456ddbebf7/info of ac5e62e4423d0e893d9b60456ddbebf7 into 42d02823b91a4131b97e5c2d827291ee(size=160.7 K), total size for store is 160.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:37,500 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:37,500 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., storeName=ac5e62e4423d0e893d9b60456ddbebf7/info, priority=13, startTime=1733343757469; duration=0sec 2024-12-04T20:22:37,500 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:37,500 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac5e62e4423d0e893d9b60456ddbebf7:info 2024-12-04T20:22:37,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/f4bc00690d434993a22f0802ac958439 2024-12-04T20:22:37,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/f4bc00690d434993a22f0802ac958439 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/f4bc00690d434993a22f0802ac958439 2024-12-04T20:22:37,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/f4bc00690d434993a22f0802ac958439, entries=12, sequenceid=292, filesize=17.5 K 2024-12-04T20:22:37,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=4.20 KB/4304 for ac5e62e4423d0e893d9b60456ddbebf7 in 434ms, sequenceid=292, compaction requested=false 2024-12-04T20:22:37,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:38,490 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:38,490 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:39,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T20:22:39,491 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:39,491 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:39,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/0f1cbdf3c48e435e82b2cadb9429819d is 1080, key is row0221/info:/1733343757472/Put/seqid=0 2024-12-04T20:22:39,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741874_1050 (size=12523) 2024-12-04T20:22:39,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741874_1050 (size=12523) 2024-12-04T20:22:39,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/0f1cbdf3c48e435e82b2cadb9429819d 2024-12-04T20:22:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/0f1cbdf3c48e435e82b2cadb9429819d as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0f1cbdf3c48e435e82b2cadb9429819d 2024-12-04T20:22:39,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0f1cbdf3c48e435e82b2cadb9429819d, entries=7, sequenceid=303, filesize=12.2 K 2024-12-04T20:22:39,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for ac5e62e4423d0e893d9b60456ddbebf7 in 27ms, sequenceid=303, compaction requested=true 2024-12-04T20:22:39,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:39,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac5e62e4423d0e893d9b60456ddbebf7:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:39,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:39,518 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:39,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-04T20:22:39,520 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 195045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:39,520 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): ac5e62e4423d0e893d9b60456ddbebf7/info is initiating minor compaction (all files) 2024-12-04T20:22:39,520 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac5e62e4423d0e893d9b60456ddbebf7/info in TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:39,520 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/42d02823b91a4131b97e5c2d827291ee, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/f4bc00690d434993a22f0802ac958439, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0f1cbdf3c48e435e82b2cadb9429819d] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp, totalSize=190.5 K 2024-12-04T20:22:39,521 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d02823b91a4131b97e5c2d827291ee, keycount=147, bloomtype=ROW, size=160.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733343737039 2024-12-04T20:22:39,521 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4bc00690d434993a22f0802ac958439, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733343757449 2024-12-04T20:22:39,522 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1cbdf3c48e435e82b2cadb9429819d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733343757472 2024-12-04T20:22:39,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/07b701df583e4a7aa1a1b8857c0d53fc is 1080, key is row0228/info:/1733343759493/Put/seqid=0 2024-12-04T20:22:39,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741875_1051 (size=17918) 2024-12-04T20:22:39,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741875_1051 (size=17918) 2024-12-04T20:22:39,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/07b701df583e4a7aa1a1b8857c0d53fc 2024-12-04T20:22:39,535 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac5e62e4423d0e893d9b60456ddbebf7#info#compaction#91 average throughput is 56.78 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:39,536 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/8a0842f0cd87450b9a6d75fcf5a80cc1 is 1080, key is row0062/info:/1733343737039/Put/seqid=0 2024-12-04T20:22:39,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/07b701df583e4a7aa1a1b8857c0d53fc as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07b701df583e4a7aa1a1b8857c0d53fc 2024-12-04T20:22:39,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741876_1052 (size=185211) 2024-12-04T20:22:39,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741876_1052 (size=185211) 2024-12-04T20:22:39,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07b701df583e4a7aa1a1b8857c0d53fc, entries=12, sequenceid=318, filesize=17.5 K 2024-12-04T20:22:39,549 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/8a0842f0cd87450b9a6d75fcf5a80cc1 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8a0842f0cd87450b9a6d75fcf5a80cc1 2024-12-04T20:22:39,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for ac5e62e4423d0e893d9b60456ddbebf7 in 30ms, sequenceid=318, compaction requested=false 2024-12-04T20:22:39,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33877 {}] regionserver.HRegion(8855): Flush requested on ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:39,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-12-04T20:22:39,555 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac5e62e4423d0e893d9b60456ddbebf7/info of ac5e62e4423d0e893d9b60456ddbebf7 into 8a0842f0cd87450b9a6d75fcf5a80cc1(size=180.9 K), total size for store is 198.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:39,555 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:39,555 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., storeName=ac5e62e4423d0e893d9b60456ddbebf7/info, priority=13, startTime=1733343759518; duration=0sec 2024-12-04T20:22:39,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/cfde329f04f14146a6be9650e12c8c5c is 1080, key is row0240/info:/1733343759521/Put/seqid=0 2024-12-04T20:22:39,555 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:39,555 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac5e62e4423d0e893d9b60456ddbebf7:info 2024-12-04T20:22:39,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741877_1053 (size=16839) 2024-12-04T20:22:39,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741877_1053 (size=16839) 2024-12-04T20:22:39,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/cfde329f04f14146a6be9650e12c8c5c 2024-12-04T20:22:39,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/cfde329f04f14146a6be9650e12c8c5c as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/cfde329f04f14146a6be9650e12c8c5c 2024-12-04T20:22:39,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/cfde329f04f14146a6be9650e12c8c5c, entries=11, sequenceid=332, filesize=16.4 K 2024-12-04T20:22:39,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=6.30 KB/6456 for ac5e62e4423d0e893d9b60456ddbebf7 in 19ms, sequenceid=332, compaction requested=true 2024-12-04T20:22:39,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:39,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac5e62e4423d0e893d9b60456ddbebf7:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T20:22:39,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:39,570 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T20:22:39,571 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 219968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T20:22:39,571 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1541): ac5e62e4423d0e893d9b60456ddbebf7/info is initiating minor compaction (all files) 2024-12-04T20:22:39,571 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac5e62e4423d0e893d9b60456ddbebf7/info in TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:39,571 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8a0842f0cd87450b9a6d75fcf5a80cc1, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07b701df583e4a7aa1a1b8857c0d53fc, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/cfde329f04f14146a6be9650e12c8c5c] into tmpdir=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp, totalSize=214.8 K 2024-12-04T20:22:39,572 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a0842f0cd87450b9a6d75fcf5a80cc1, keycount=166, bloomtype=ROW, size=180.9 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733343737039 2024-12-04T20:22:39,572 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07b701df583e4a7aa1a1b8857c0d53fc, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733343759493 2024-12-04T20:22:39,572 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfde329f04f14146a6be9650e12c8c5c, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733343759521 2024-12-04T20:22:39,585 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac5e62e4423d0e893d9b60456ddbebf7#info#compaction#93 average throughput is 64.65 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:22:39,586 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/15bd13ad4f744b81a176ba9513f7f21f is 1080, key is row0062/info:/1733343737039/Put/seqid=0 2024-12-04T20:22:39,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741878_1054 (size=210207) 2024-12-04T20:22:39,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741878_1054 (size=210207) 2024-12-04T20:22:39,593 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/15bd13ad4f744b81a176ba9513f7f21f as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/15bd13ad4f744b81a176ba9513f7f21f 2024-12-04T20:22:39,599 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac5e62e4423d0e893d9b60456ddbebf7/info of ac5e62e4423d0e893d9b60456ddbebf7 into 15bd13ad4f744b81a176ba9513f7f21f(size=205.3 K), total size for store is 205.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:22:39,599 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:39,599 INFO [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., storeName=ac5e62e4423d0e893d9b60456ddbebf7/info, priority=13, startTime=1733343759570; duration=0sec 2024-12-04T20:22:39,599 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T20:22:39,599 DEBUG [RS:0;c2ef38372881:33877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac5e62e4423d0e893d9b60456ddbebf7:info 2024-12-04T20:22:40,492 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:40,492 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:41,494 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:41,494 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:41,564 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-12-04T20:22:41,565 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33877%2C1733343723612.1733343761564 2024-12-04T20:22:41,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,575 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.1733343724388 with entries=318, filesize=310.38 KB; new WAL /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.1733343761564 2024-12-04T20:22:41,576 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43321:43321),(127.0.0.1/127.0.0.1:37933:37933)] 2024-12-04T20:22:41,577 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.1733343724388 is not closed yet, will try archiving it next time 2024-12-04T20:22:41,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741833_1009 (size=317837) 2024-12-04T20:22:41,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741833_1009 (size=317837) 2024-12-04T20:22:41,583 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 007562742faf9522bcb062faed9d6eae: 2024-12-04T20:22:41,583 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ac5e62e4423d0e893d9b60456ddbebf7 1/1 column families, dataSize=6.30 KB heapSize=7 KB 2024-12-04T20:22:41,588 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/8b599355914a475587a86754114ed697 is 1080, key is row0251/info:/1733343759552/Put/seqid=0 2024-12-04T20:22:41,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741880_1056 (size=11436) 2024-12-04T20:22:41,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741880_1056 (size=11436) 2024-12-04T20:22:41,593 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/8b599355914a475587a86754114ed697 2024-12-04T20:22:41,597 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/.tmp/info/8b599355914a475587a86754114ed697 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8b599355914a475587a86754114ed697 2024-12-04T20:22:41,603 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8b599355914a475587a86754114ed697, entries=6, sequenceid=343, filesize=11.2 K 2024-12-04T20:22:41,604 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for ac5e62e4423d0e893d9b60456ddbebf7 in 21ms, sequenceid=343, compaction requested=false 2024-12-04T20:22:41,604 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ac5e62e4423d0e893d9b60456ddbebf7: 2024-12-04T20:22:41,604 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB 2024-12-04T20:22:41,609 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/info/532734559931448d9b456c7107eefd39 is 186, key is TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae./info:regioninfo/1733343739985/Put/seqid=0 2024-12-04T20:22:41,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741881_1057 (size=6153) 2024-12-04T20:22:41,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741881_1057 (size=6153) 2024-12-04T20:22:41,614 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/info/532734559931448d9b456c7107eefd39 2024-12-04T20:22:41,619 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/.tmp/info/532734559931448d9b456c7107eefd39 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/info/532734559931448d9b456c7107eefd39 2024-12-04T20:22:41,623 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/info/532734559931448d9b456c7107eefd39, entries=5, sequenceid=21, filesize=6.0 K 2024-12-04T20:22:41,624 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 20ms, sequenceid=21, compaction requested=false 2024-12-04T20:22:41,624 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-04T20:22:41,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C33877%2C1733343723612.1733343761624 2024-12-04T20:22:41,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,629 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.1733343761564 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.1733343761624 2024-12-04T20:22:41,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741879_1055 (size=731) 2024-12-04T20:22:41,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741879_1055 (size=731) 2024-12-04T20:22:41,631 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43321:43321),(127.0.0.1/127.0.0.1:37933:37933)] 2024-12-04T20:22:41,631 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.1733343724388 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/oldWALs/c2ef38372881%2C33877%2C1733343723612.1733343724388 2024-12-04T20:22:41,632 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T20:22:41,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:22:41,632 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:22:41,632 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:41,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:41,632 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/WALs/c2ef38372881,33877,1733343723612/c2ef38372881%2C33877%2C1733343723612.1733343761564 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/oldWALs/c2ef38372881%2C33877%2C1733343723612.1733343761564 2024-12-04T20:22:41,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:22:41,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:41,632 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:22:41,632 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1634437415, stopped=false 2024-12-04T20:22:41,633 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,34749,1733343723571 2024-12-04T20:22:41,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:41,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:41,634 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:22:41,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:41,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:41,634 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:41,634 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:22:41,634 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:41,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:41,634 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:41,634 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,33877,1733343723612' ***** 2024-12-04T20:22:41,634 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:22:41,635 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(3091): Received CLOSE for 007562742faf9522bcb062faed9d6eae 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(3091): Received CLOSE for ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,33877,1733343723612 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:33877. 2024-12-04T20:22:41,635 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 007562742faf9522bcb062faed9d6eae, disabling compactions & flushes 2024-12-04T20:22:41,635 DEBUG [RS:0;c2ef38372881:33877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:41,635 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:41,635 DEBUG [RS:0;c2ef38372881:33877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:41,635 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:41,635 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. after waiting 0 ms 2024-12-04T20:22:41,635 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:22:41,635 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:22:41,636 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-04T20:22:41,636 DEBUG [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1325): Online Regions={007562742faf9522bcb062faed9d6eae=TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae., ac5e62e4423d0e893d9b60456ddbebf7=TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T20:22:41,636 DEBUG [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1351): Waiting on 007562742faf9522bcb062faed9d6eae, 1588230740, ac5e62e4423d0e893d9b60456ddbebf7 2024-12-04T20:22:41,636 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:22:41,636 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:22:41,636 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:22:41,636 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:22:41,636 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:22:41,636 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62->hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f-bottom] to archive 2024-12-04T20:22:41,636 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:22:41,638 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:41,638 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c2ef38372881:34749 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T20:22:41,638 WARN [StoreCloser-TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-12-04T20:22:41,640 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-04T20:22:41,640 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:22:41,640 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:22:41,640 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343761636Running coprocessor pre-close hooks at 1733343761636Disabling compacts and flushes for region at 1733343761636Disabling writes for close at 1733343761636Writing region close event to WAL at 1733343761637 (+1 ms)Running coprocessor post-close hooks at 1733343761640 (+3 ms)Closed at 1733343761640 2024-12-04T20:22:41,640 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:22:41,641 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/007562742faf9522bcb062faed9d6eae/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130 2024-12-04T20:22:41,642 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:41,642 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 007562742faf9522bcb062faed9d6eae: Waiting for close lock at 1733343761635Running coprocessor pre-close hooks at 1733343761635Disabling compacts and flushes for region at 1733343761635Disabling writes for close at 1733343761635Writing region close event to WAL at 1733343761639 (+4 ms)Running coprocessor post-close hooks at 1733343761642 (+3 ms)Closed at 1733343761642 2024-12-04T20:22:41,642 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733343739187.007562742faf9522bcb062faed9d6eae. 2024-12-04T20:22:41,642 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ac5e62e4423d0e893d9b60456ddbebf7, disabling compactions & flushes 2024-12-04T20:22:41,642 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:41,642 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:41,642 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. after waiting 0 ms 2024-12-04T20:22:41,642 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:41,643 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62->hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/1895d4fe5ab48cac297c933e9fe0af62/info/e39f40ccb6f847a4b881063f1608d56f-top, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-70a0413710f44bfcb00f5deaa9086a2c, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fce955b8d0454f7f9962c2cf5a00ba1c, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-29ea0eabff9b478981ea9313060a54d8, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1573df7e20b946b193c4e138c88d3d9f, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0df0f13920614451ae68ec93382039bd, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/686aad72a6414639ab14fb0033fe4983, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07885ee874e54d10bec8f7bb7962ed31, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/4ece84ce22cc485a8cd187267045cf1d, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/5e7fcf102bb5468f85b87c3cff5851d4, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/3731cd62ab3c4d9e873fd9a18b5368ff, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a2be8683b66749a79470d2a5d5efc9aa, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fa550ae2a9824d32a4da4d4dfdb7cc55, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/35edc55385324eaf8f2eb23ffbca3841, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8f832ccb477d4c938ec57343bee375af, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/15d2ef595e534f3888c5c86e295622da, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1adf88a92b224b73ac36e83383c2c679, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/42d02823b91a4131b97e5c2d827291ee, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a6c41495b3964e4b8c7cb31d84d71332, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/f4bc00690d434993a22f0802ac958439, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8a0842f0cd87450b9a6d75fcf5a80cc1, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0f1cbdf3c48e435e82b2cadb9429819d, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07b701df583e4a7aa1a1b8857c0d53fc, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/cfde329f04f14146a6be9650e12c8c5c] to archive 2024-12-04T20:22:41,644 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:22:41,645 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/e39f40ccb6f847a4b881063f1608d56f.1895d4fe5ab48cac297c933e9fe0af62 2024-12-04T20:22:41,646 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-70a0413710f44bfcb00f5deaa9086a2c to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-70a0413710f44bfcb00f5deaa9086a2c 2024-12-04T20:22:41,648 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fce955b8d0454f7f9962c2cf5a00ba1c to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fce955b8d0454f7f9962c2cf5a00ba1c 2024-12-04T20:22:41,649 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-29ea0eabff9b478981ea9313060a54d8 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/TestLogRolling-testLogRolling=1895d4fe5ab48cac297c933e9fe0af62-29ea0eabff9b478981ea9313060a54d8 2024-12-04T20:22:41,650 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1573df7e20b946b193c4e138c88d3d9f to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1573df7e20b946b193c4e138c88d3d9f 2024-12-04T20:22:41,651 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0df0f13920614451ae68ec93382039bd to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0df0f13920614451ae68ec93382039bd 2024-12-04T20:22:41,652 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/686aad72a6414639ab14fb0033fe4983 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/686aad72a6414639ab14fb0033fe4983 2024-12-04T20:22:41,653 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07885ee874e54d10bec8f7bb7962ed31 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07885ee874e54d10bec8f7bb7962ed31 2024-12-04T20:22:41,654 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/4ece84ce22cc485a8cd187267045cf1d to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/4ece84ce22cc485a8cd187267045cf1d 2024-12-04T20:22:41,655 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/5e7fcf102bb5468f85b87c3cff5851d4 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/5e7fcf102bb5468f85b87c3cff5851d4 2024-12-04T20:22:41,655 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/3731cd62ab3c4d9e873fd9a18b5368ff to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/3731cd62ab3c4d9e873fd9a18b5368ff 2024-12-04T20:22:41,656 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a2be8683b66749a79470d2a5d5efc9aa to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a2be8683b66749a79470d2a5d5efc9aa 2024-12-04T20:22:41,657 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fa550ae2a9824d32a4da4d4dfdb7cc55 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/fa550ae2a9824d32a4da4d4dfdb7cc55 2024-12-04T20:22:41,658 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/35edc55385324eaf8f2eb23ffbca3841 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/35edc55385324eaf8f2eb23ffbca3841 2024-12-04T20:22:41,659 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8f832ccb477d4c938ec57343bee375af to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8f832ccb477d4c938ec57343bee375af 2024-12-04T20:22:41,659 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/15d2ef595e534f3888c5c86e295622da to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/15d2ef595e534f3888c5c86e295622da 2024-12-04T20:22:41,660 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1adf88a92b224b73ac36e83383c2c679 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/1adf88a92b224b73ac36e83383c2c679 2024-12-04T20:22:41,661 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/42d02823b91a4131b97e5c2d827291ee to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/42d02823b91a4131b97e5c2d827291ee 2024-12-04T20:22:41,662 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a6c41495b3964e4b8c7cb31d84d71332 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/a6c41495b3964e4b8c7cb31d84d71332 2024-12-04T20:22:41,663 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/f4bc00690d434993a22f0802ac958439 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/f4bc00690d434993a22f0802ac958439 2024-12-04T20:22:41,664 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8a0842f0cd87450b9a6d75fcf5a80cc1 to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/8a0842f0cd87450b9a6d75fcf5a80cc1 2024-12-04T20:22:41,665 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0f1cbdf3c48e435e82b2cadb9429819d to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/0f1cbdf3c48e435e82b2cadb9429819d 2024-12-04T20:22:41,665 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07b701df583e4a7aa1a1b8857c0d53fc to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/07b701df583e4a7aa1a1b8857c0d53fc 2024-12-04T20:22:41,666 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/cfde329f04f14146a6be9650e12c8c5c to hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/archive/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/info/cfde329f04f14146a6be9650e12c8c5c 2024-12-04T20:22:41,666 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [fce955b8d0454f7f9962c2cf5a00ba1c=42984, 1573df7e20b946b193c4e138c88d3d9f=12516, 0df0f13920614451ae68ec93382039bd=64714, 686aad72a6414639ab14fb0033fe4983=19000, 07885ee874e54d10bec8f7bb7962ed31=17906, 4ece84ce22cc485a8cd187267045cf1d=85371, 5e7fcf102bb5468f85b87c3cff5851d4=12516, 3731cd62ab3c4d9e873fd9a18b5368ff=19000, a2be8683b66749a79470d2a5d5efc9aa=110275, fa550ae2a9824d32a4da4d4dfdb7cc55=15750, 35edc55385324eaf8f2eb23ffbca3841=12516, 8f832ccb477d4c938ec57343bee375af=132069, 15d2ef595e534f3888c5c86e295622da=19000, 1adf88a92b224b73ac36e83383c2c679=21157, 42d02823b91a4131b97e5c2d827291ee=164604, a6c41495b3964e4b8c7cb31d84d71332=21171, f4bc00690d434993a22f0802ac958439=17918, 8a0842f0cd87450b9a6d75fcf5a80cc1=185211, 0f1cbdf3c48e435e82b2cadb9429819d=12523, 07b701df583e4a7aa1a1b8857c0d53fc=17918, cfde329f04f14146a6be9650e12c8c5c=16839] 2024-12-04T20:22:41,669 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/data/default/TestLogRolling-testLogRolling/ac5e62e4423d0e893d9b60456ddbebf7/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=130 2024-12-04T20:22:41,670 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:41,670 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ac5e62e4423d0e893d9b60456ddbebf7: Waiting for close lock at 1733343761642Running coprocessor pre-close hooks at 1733343761642Disabling compacts and flushes for region at 1733343761642Disabling writes for close at 1733343761642Writing region close event to WAL at 1733343761667 (+25 ms)Running coprocessor post-close hooks at 1733343761670 (+3 ms)Closed at 1733343761670 2024-12-04T20:22:41,670 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733343739187.ac5e62e4423d0e893d9b60456ddbebf7. 2024-12-04T20:22:41,836 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,33877,1733343723612; all regions closed. 2024-12-04T20:22:41,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741834_1010 (size=8107) 2024-12-04T20:22:41,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741834_1010 (size=8107) 2024-12-04T20:22:41,845 DEBUG [RS:0;c2ef38372881:33877 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/oldWALs 2024-12-04T20:22:41,845 INFO [RS:0;c2ef38372881:33877 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C33877%2C1733343723612.meta:.meta(num 1733343724757) 2024-12-04T20:22:41,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741882_1058 (size=780) 2024-12-04T20:22:41,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741882_1058 (size=780) 2024-12-04T20:22:41,851 DEBUG [RS:0;c2ef38372881:33877 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/oldWALs 2024-12-04T20:22:41,851 INFO [RS:0;c2ef38372881:33877 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C33877%2C1733343723612:(num 1733343761624) 2024-12-04T20:22:41,851 DEBUG [RS:0;c2ef38372881:33877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:41,851 INFO [RS:0;c2ef38372881:33877 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:22:41,852 INFO [RS:0;c2ef38372881:33877 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:22:41,852 INFO [RS:0;c2ef38372881:33877 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T20:22:41,852 INFO [RS:0;c2ef38372881:33877 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:22:41,852 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:22:41,852 INFO [RS:0;c2ef38372881:33877 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33877 2024-12-04T20:22:41,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,33877,1733343723612 2024-12-04T20:22:41,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:22:41,854 INFO [RS:0;c2ef38372881:33877 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:22:41,855 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,33877,1733343723612] 2024-12-04T20:22:41,855 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,33877,1733343723612 already deleted, retry=false 2024-12-04T20:22:41,856 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,33877,1733343723612 expired; onlineServers=0 2024-12-04T20:22:41,856 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,34749,1733343723571' ***** 2024-12-04T20:22:41,856 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:22:41,856 INFO [M:0;c2ef38372881:34749 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:22:41,856 INFO [M:0;c2ef38372881:34749 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:22:41,856 DEBUG [M:0;c2ef38372881:34749 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:22:41,856 DEBUG [M:0;c2ef38372881:34749 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:22:41,856 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:22:41,856 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343724157 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343724157,5,FailOnTimeoutGroup] 2024-12-04T20:22:41,856 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343724157 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343724157,5,FailOnTimeoutGroup] 2024-12-04T20:22:41,856 INFO [M:0;c2ef38372881:34749 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:22:41,856 INFO [M:0;c2ef38372881:34749 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:22:41,856 DEBUG [M:0;c2ef38372881:34749 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:22:41,856 INFO [M:0;c2ef38372881:34749 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:22:41,857 INFO [M:0;c2ef38372881:34749 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:22:41,857 INFO [M:0;c2ef38372881:34749 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:22:41,857 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:22:41,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:22:41,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:41,857 DEBUG [M:0;c2ef38372881:34749 {}] zookeeper.ZKUtil(347): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:22:41,857 WARN [M:0;c2ef38372881:34749 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:22:41,858 INFO [M:0;c2ef38372881:34749 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/.lastflushedseqids 2024-12-04T20:22:41,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741883_1059 (size=228) 2024-12-04T20:22:41,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741883_1059 (size=228) 2024-12-04T20:22:41,864 INFO [M:0;c2ef38372881:34749 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:22:41,864 INFO [M:0;c2ef38372881:34749 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:22:41,864 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:22:41,864 INFO [M:0;c2ef38372881:34749 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:41,864 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:41,864 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:22:41,864 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:41,865 INFO [M:0;c2ef38372881:34749 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.42 KB heapSize=63.36 KB 2024-12-04T20:22:41,880 DEBUG [M:0;c2ef38372881:34749 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f7f224ccce324c6bb6d0615183fbeee2 is 82, key is hbase:meta,,1/info:regioninfo/1733343724777/Put/seqid=0 2024-12-04T20:22:41,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741884_1060 (size=5672) 2024-12-04T20:22:41,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741884_1060 (size=5672) 2024-12-04T20:22:41,884 INFO [M:0;c2ef38372881:34749 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f7f224ccce324c6bb6d0615183fbeee2 2024-12-04T20:22:41,903 DEBUG [M:0;c2ef38372881:34749 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0831e5ad2ae433a839d6c6073da0ccf is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733343725229/Put/seqid=0 2024-12-04T20:22:41,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741885_1061 (size=7089) 2024-12-04T20:22:41,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741885_1061 (size=7089) 2024-12-04T20:22:41,908 INFO [M:0;c2ef38372881:34749 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.81 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0831e5ad2ae433a839d6c6073da0ccf 2024-12-04T20:22:41,911 INFO [M:0;c2ef38372881:34749 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f0831e5ad2ae433a839d6c6073da0ccf 2024-12-04T20:22:41,926 DEBUG [M:0;c2ef38372881:34749 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/740dcaa52e8a491f82a3b3661bd9de3b is 69, key is c2ef38372881,33877,1733343723612/rs:state/1733343724247/Put/seqid=0 2024-12-04T20:22:41,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741886_1062 (size=5156) 2024-12-04T20:22:41,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741886_1062 (size=5156) 2024-12-04T20:22:41,931 INFO [M:0;c2ef38372881:34749 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/740dcaa52e8a491f82a3b3661bd9de3b 2024-12-04T20:22:41,953 DEBUG [M:0;c2ef38372881:34749 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5313cf63e76f4ee28fb6b8cf49faaced is 52, key is load_balancer_on/state:d/1733343724841/Put/seqid=0 2024-12-04T20:22:41,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:41,955 INFO [RS:0;c2ef38372881:33877 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:22:41,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33877-0x100a6e59c8f0001, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:41,955 INFO [RS:0;c2ef38372881:33877 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,33877,1733343723612; zookeeper connection closed. 2024-12-04T20:22:41,955 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@339a958e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@339a958e 2024-12-04T20:22:41,956 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T20:22:41,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741887_1063 (size=5056) 2024-12-04T20:22:41,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741887_1063 (size=5056) 2024-12-04T20:22:41,958 INFO [M:0;c2ef38372881:34749 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5313cf63e76f4ee28fb6b8cf49faaced 2024-12-04T20:22:41,963 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f7f224ccce324c6bb6d0615183fbeee2 as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f7f224ccce324c6bb6d0615183fbeee2 2024-12-04T20:22:41,967 INFO [M:0;c2ef38372881:34749 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f7f224ccce324c6bb6d0615183fbeee2, entries=8, sequenceid=125, filesize=5.5 K 2024-12-04T20:22:41,968 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0831e5ad2ae433a839d6c6073da0ccf as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f0831e5ad2ae433a839d6c6073da0ccf 2024-12-04T20:22:41,973 INFO [M:0;c2ef38372881:34749 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f0831e5ad2ae433a839d6c6073da0ccf 2024-12-04T20:22:41,973 INFO [M:0;c2ef38372881:34749 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f0831e5ad2ae433a839d6c6073da0ccf, entries=13, sequenceid=125, filesize=6.9 K 2024-12-04T20:22:41,974 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/740dcaa52e8a491f82a3b3661bd9de3b as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/740dcaa52e8a491f82a3b3661bd9de3b 2024-12-04T20:22:41,978 INFO [M:0;c2ef38372881:34749 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/740dcaa52e8a491f82a3b3661bd9de3b, entries=1, sequenceid=125, filesize=5.0 K 2024-12-04T20:22:41,979 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5313cf63e76f4ee28fb6b8cf49faaced as hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5313cf63e76f4ee28fb6b8cf49faaced 2024-12-04T20:22:41,982 INFO [M:0;c2ef38372881:34749 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44207/user/jenkins/test-data/9e72d0a6-fd49-3ef6-f65d-a9c260df9e2a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5313cf63e76f4ee28fb6b8cf49faaced, entries=1, sequenceid=125, filesize=4.9 K 2024-12-04T20:22:41,983 INFO [M:0;c2ef38372881:34749 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=125, compaction requested=false 2024-12-04T20:22:41,985 INFO [M:0;c2ef38372881:34749 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:41,985 DEBUG [M:0;c2ef38372881:34749 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343761864Disabling compacts and flushes for region at 1733343761864Disabling writes for close at 1733343761864Obtaining lock to block concurrent updates at 1733343761865 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733343761865Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52651, getHeapSize=64816, getOffHeapSize=0, getCellsCount=148 at 1733343761865Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733343761866 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733343761866Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733343761879 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733343761879Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733343761888 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733343761902 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733343761903 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733343761911 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733343761925 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733343761925Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733343761935 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733343761953 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733343761953Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e3056c0: reopening flushed file at 1733343761962 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@476cac6: reopening flushed file at 1733343761967 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f6963ac: reopening flushed file at 1733343761973 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b529507: reopening flushed file at 1733343761978 (+5 ms)Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=125, compaction requested=false at 1733343761983 (+5 ms)Writing region close event to WAL at 1733343761984 (+1 ms)Closed at 1733343761984 2024-12-04T20:22:41,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:41,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34413 is added to blk_1073741830_1006 (size=61320) 2024-12-04T20:22:41,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35743 is added to blk_1073741830_1006 (size=61320) 2024-12-04T20:22:41,987 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:22:41,987 INFO [M:0;c2ef38372881:34749 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:22:41,987 INFO [M:0;c2ef38372881:34749 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34749 2024-12-04T20:22:41,988 INFO [M:0;c2ef38372881:34749 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:22:42,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:42,089 INFO [M:0;c2ef38372881:34749 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:22:42,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34749-0x100a6e59c8f0000, quorum=127.0.0.1:60207, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:42,097 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@191b8d86{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:42,099 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5689196f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:42,099 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:42,099 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@132e4184{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:42,099 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@605d7cdf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:42,102 WARN [BP-965759213-172.17.0.2-1733343722982 heartbeating to localhost/127.0.0.1:44207 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:22:42,102 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:22:42,102 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:22:42,102 WARN [BP-965759213-172.17.0.2-1733343722982 heartbeating to localhost/127.0.0.1:44207 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-965759213-172.17.0.2-1733343722982 (Datanode Uuid fc4d7e90-87c0-4126-b320-9db3129140b6) service to localhost/127.0.0.1:44207 2024-12-04T20:22:42,103 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/data/data3/current/BP-965759213-172.17.0.2-1733343722982 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:42,103 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/data/data4/current/BP-965759213-172.17.0.2-1733343722982 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:42,103 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:22:42,105 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26d44036{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:42,105 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d8c7847{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:42,105 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:42,105 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ceb85b6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:42,105 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63e2e387{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:42,107 WARN [BP-965759213-172.17.0.2-1733343722982 heartbeating to localhost/127.0.0.1:44207 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:22:42,107 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:22:42,107 WARN [BP-965759213-172.17.0.2-1733343722982 heartbeating to localhost/127.0.0.1:44207 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-965759213-172.17.0.2-1733343722982 (Datanode Uuid 54c70cc2-697f-46c6-bc12-fc6ffbe70312) service to localhost/127.0.0.1:44207 2024-12-04T20:22:42,107 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:22:42,107 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/data/data1/current/BP-965759213-172.17.0.2-1733343722982 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:42,107 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/cluster_61c7c4d6-f192-114d-589c-887437a07063/data/data2/current/BP-965759213-172.17.0.2-1733343722982 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:42,108 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:22:42,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d900229{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:22:42,113 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@62c26df6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:42,113 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:42,114 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@110e33ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:42,114 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a4214d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:42,119 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:22:42,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T20:22:42,155 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=233 (was 209) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44207 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44207 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44207 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44207 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44207 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44207 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44207 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:44207 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=512 (was 485) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=78 (was 119), ProcessCount=11 (was 11), AvailableMemoryMB=4729 (was 4380) - AvailableMemoryMB LEAK? - 2024-12-04T20:22:42,162 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=233, OpenFileDescriptor=512, MaxFileDescriptor=1048576, SystemLoadAverage=78, ProcessCount=11, AvailableMemoryMB=4729 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.log.dir so I do NOT create it in target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/174d236c-379a-00a7-67b3-4615dc05dff8/hadoop.tmp.dir so I do NOT create it in target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649, deleteOnExit=true 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/test.cache.data in system properties and HBase conf 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:22:42,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:22:42,163 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:22:42,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:22:42,176 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:22:42,213 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:22:42,216 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:22:42,220 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:22:42,220 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:22:42,220 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:22:42,220 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:22:42,221 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a024a47{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:22:42,221 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ee125cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:22:42,258 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:22:42,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10c1adfc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/java.io.tmpdir/jetty-localhost-36489-hadoop-hdfs-3_4_1-tests_jar-_-any-2212591072008080624/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:22:42,309 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5fab3ee3{HTTP/1.1, (http/1.1)}{localhost:36489} 2024-12-04T20:22:42,309 INFO [Time-limited test {}] server.Server(415): Started @274149ms 2024-12-04T20:22:42,320 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T20:22:42,353 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:22:42,355 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:22:42,356 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:22:42,356 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:22:42,356 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:22:42,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20042785{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:22:42,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d148abe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:22:42,449 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4578d3a2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/java.io.tmpdir/jetty-localhost-38803-hadoop-hdfs-3_4_1-tests_jar-_-any-14005850694279691067/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:42,449 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@259cffcc{HTTP/1.1, (http/1.1)}{localhost:38803} 2024-12-04T20:22:42,449 INFO [Time-limited test {}] server.Server(415): Started @274289ms 2024-12-04T20:22:42,450 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:22:42,474 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:22:42,476 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:22:42,477 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:22:42,477 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:22:42,477 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:22:42,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e818af2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:22:42,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2578bc63{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:22:42,494 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:42,494 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:42,508 WARN [Thread-2506 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/data/data2/current/BP-645053777-172.17.0.2-1733343762179/current, will proceed with Du for space computation calculation, 2024-12-04T20:22:42,508 WARN [Thread-2505 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/data/data1/current/BP-645053777-172.17.0.2-1733343762179/current, will proceed with Du for space computation calculation, 2024-12-04T20:22:42,526 WARN [Thread-2484 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:22:42,528 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad9c69bed8f9279e with lease ID 0x88e829322c97f527: Processing first storage report for DS-f09ad05b-4b36-47d1-bf40-921ea0590c5d from datanode DatanodeRegistration(127.0.0.1:33475, datanodeUuid=b61205d4-cef8-4a03-ac4c-ba7d82def94d, infoPort=38423, infoSecurePort=0, ipcPort=40945, storageInfo=lv=-57;cid=testClusterID;nsid=1767651119;c=1733343762179) 2024-12-04T20:22:42,528 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad9c69bed8f9279e with lease ID 0x88e829322c97f527: from storage DS-f09ad05b-4b36-47d1-bf40-921ea0590c5d node DatanodeRegistration(127.0.0.1:33475, datanodeUuid=b61205d4-cef8-4a03-ac4c-ba7d82def94d, infoPort=38423, infoSecurePort=0, ipcPort=40945, storageInfo=lv=-57;cid=testClusterID;nsid=1767651119;c=1733343762179), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:22:42,529 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad9c69bed8f9279e with lease ID 0x88e829322c97f527: Processing first storage report for DS-509a7692-1bc6-4700-9116-27d83189c42b from datanode DatanodeRegistration(127.0.0.1:33475, datanodeUuid=b61205d4-cef8-4a03-ac4c-ba7d82def94d, infoPort=38423, infoSecurePort=0, ipcPort=40945, storageInfo=lv=-57;cid=testClusterID;nsid=1767651119;c=1733343762179) 2024-12-04T20:22:42,529 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad9c69bed8f9279e with lease ID 0x88e829322c97f527: from storage DS-509a7692-1bc6-4700-9116-27d83189c42b node DatanodeRegistration(127.0.0.1:33475, datanodeUuid=b61205d4-cef8-4a03-ac4c-ba7d82def94d, infoPort=38423, infoSecurePort=0, ipcPort=40945, storageInfo=lv=-57;cid=testClusterID;nsid=1767651119;c=1733343762179), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:22:42,578 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@413a6699{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/java.io.tmpdir/jetty-localhost-38241-hadoop-hdfs-3_4_1-tests_jar-_-any-12681600075371383751/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:42,578 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1cfa6b2d{HTTP/1.1, (http/1.1)}{localhost:38241} 2024-12-04T20:22:42,578 INFO [Time-limited test {}] server.Server(415): Started @274418ms 2024-12-04T20:22:42,579 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:22:42,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:22:42,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:22:42,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T20:22:42,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-12-04T20:22:42,630 WARN [Thread-2531 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/data/data3/current/BP-645053777-172.17.0.2-1733343762179/current, will proceed with Du for space computation calculation, 2024-12-04T20:22:42,631 WARN [Thread-2532 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/data/data4/current/BP-645053777-172.17.0.2-1733343762179/current, will proceed with Du for space computation calculation, 2024-12-04T20:22:42,648 WARN [Thread-2520 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:22:42,650 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd527257ed0e670ea with lease ID 0x88e829322c97f528: Processing first storage report for DS-fdddf916-f8d5-4da5-bc83-657357fb5139 from datanode DatanodeRegistration(127.0.0.1:39815, datanodeUuid=1f39bd39-b261-49f3-97a3-badc5e6c22b2, infoPort=44001, infoSecurePort=0, ipcPort=43415, storageInfo=lv=-57;cid=testClusterID;nsid=1767651119;c=1733343762179) 2024-12-04T20:22:42,650 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd527257ed0e670ea with lease ID 0x88e829322c97f528: from storage DS-fdddf916-f8d5-4da5-bc83-657357fb5139 node DatanodeRegistration(127.0.0.1:39815, datanodeUuid=1f39bd39-b261-49f3-97a3-badc5e6c22b2, infoPort=44001, infoSecurePort=0, ipcPort=43415, storageInfo=lv=-57;cid=testClusterID;nsid=1767651119;c=1733343762179), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T20:22:42,650 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd527257ed0e670ea with lease ID 0x88e829322c97f528: Processing first storage report for DS-4e852776-21e2-4b4f-a348-b215a58668b3 from datanode DatanodeRegistration(127.0.0.1:39815, datanodeUuid=1f39bd39-b261-49f3-97a3-badc5e6c22b2, infoPort=44001, infoSecurePort=0, ipcPort=43415, storageInfo=lv=-57;cid=testClusterID;nsid=1767651119;c=1733343762179) 2024-12-04T20:22:42,650 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd527257ed0e670ea with lease ID 0x88e829322c97f528: from storage DS-4e852776-21e2-4b4f-a348-b215a58668b3 node DatanodeRegistration(127.0.0.1:39815, datanodeUuid=1f39bd39-b261-49f3-97a3-badc5e6c22b2, infoPort=44001, infoSecurePort=0, ipcPort=43415, storageInfo=lv=-57;cid=testClusterID;nsid=1767651119;c=1733343762179), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:22:42,696 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2 2024-12-04T20:22:42,698 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/zookeeper_0, clientPort=54065, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:22:42,699 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54065 2024-12-04T20:22:42,699 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:42,700 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:42,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:22:42,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:22:42,709 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea with version=8 2024-12-04T20:22:42,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43707/user/jenkins/test-data/c0bed26c-89d8-aafc-ae6b-85238a4b5ac9/hbase-staging 2024-12-04T20:22:42,711 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:22:42,711 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:42,711 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:42,711 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:22:42,711 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:42,711 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:22:42,711 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:22:42,711 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:22:42,712 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43977 2024-12-04T20:22:42,713 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43977 connecting to ZooKeeper ensemble=127.0.0.1:54065 2024-12-04T20:22:42,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:439770x0, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:22:42,716 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43977-0x100a6e635780000 connected 2024-12-04T20:22:42,727 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:42,729 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:42,731 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:42,731 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea, hbase.cluster.distributed=false 2024-12-04T20:22:42,733 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:22:42,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43977 2024-12-04T20:22:42,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43977 2024-12-04T20:22:42,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43977 2024-12-04T20:22:42,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43977 2024-12-04T20:22:42,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43977 2024-12-04T20:22:42,751 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:22:42,751 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:42,751 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:42,751 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:22:42,751 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:22:42,751 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:22:42,751 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:22:42,751 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:22:42,752 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41477 2024-12-04T20:22:42,753 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41477 connecting to ZooKeeper ensemble=127.0.0.1:54065 2024-12-04T20:22:42,753 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:42,754 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:42,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414770x0, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:22:42,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41477-0x100a6e635780001 connected 2024-12-04T20:22:42,758 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:42,758 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:22:42,758 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:22:42,759 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:22:42,760 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:22:42,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41477 2024-12-04T20:22:42,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41477 2024-12-04T20:22:42,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41477 2024-12-04T20:22:42,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41477 2024-12-04T20:22:42,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41477 2024-12-04T20:22:42,772 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:43977 2024-12-04T20:22:42,772 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,43977,1733343762710 2024-12-04T20:22:42,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:22:42,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:22:42,774 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,43977,1733343762710 2024-12-04T20:22:42,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:22:42,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,775 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:22:42,775 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,43977,1733343762710 from backup master directory 2024-12-04T20:22:42,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,43977,1733343762710 2024-12-04T20:22:42,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:22:42,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:22:42,776 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:22:42,776 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,43977,1733343762710 2024-12-04T20:22:42,779 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/hbase.id] with ID: b77f3145-6f48-4b4a-b931-3029a9059325 2024-12-04T20:22:42,779 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/.tmp/hbase.id 2024-12-04T20:22:42,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:22:42,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:22:42,784 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/.tmp/hbase.id]:[hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/hbase.id] 2024-12-04T20:22:42,794 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:42,794 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:22:42,795 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T20:22:42,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:22:42,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:22:42,803 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:22:42,803 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:22:42,804 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:22:42,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:22:42,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:22:42,811 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store 2024-12-04T20:22:42,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:22:42,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:22:42,819 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:42,819 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:22:42,819 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:42,819 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:42,819 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:22:42,819 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:42,819 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:42,819 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343762819Disabling compacts and flushes for region at 1733343762819Disabling writes for close at 1733343762819Writing region close event to WAL at 1733343762819Closed at 1733343762819 2024-12-04T20:22:42,820 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/.initializing 2024-12-04T20:22:42,820 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/WALs/c2ef38372881,43977,1733343762710 2024-12-04T20:22:42,822 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C43977%2C1733343762710, suffix=, logDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/WALs/c2ef38372881,43977,1733343762710, archiveDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/oldWALs, maxLogs=10 2024-12-04T20:22:42,822 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C43977%2C1733343762710.1733343762822 2024-12-04T20:22:42,827 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/WALs/c2ef38372881,43977,1733343762710/c2ef38372881%2C43977%2C1733343762710.1733343762822 2024-12-04T20:22:42,828 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38423:38423),(127.0.0.1/127.0.0.1:44001:44001)] 2024-12-04T20:22:42,829 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:22:42,829 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:42,829 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,829 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:22:42,833 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:42,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:22:42,835 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:42,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:22:42,836 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:42,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:22:42,837 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:22:42,837 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,838 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,838 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,839 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,839 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,839 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:22:42,840 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:22:42,842 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:22:42,842 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=695166, jitterRate=-0.11605097353458405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:22:42,843 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733343762829Initializing all the Stores at 1733343762830 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343762830Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343762832 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343762832Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343762832Cleaning up temporary data from old regions at 1733343762839 (+7 ms)Region opened successfully at 1733343762843 (+4 ms) 2024-12-04T20:22:42,843 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:22:42,845 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f4ca78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:22:42,846 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:22:42,846 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:22:42,846 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:22:42,846 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:22:42,847 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T20:22:42,847 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T20:22:42,847 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:22:42,849 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:22:42,849 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:22:42,850 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:22:42,850 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:22:42,851 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:22:42,851 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:22:42,851 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:22:42,852 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:22:42,853 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:22:42,854 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:22:42,854 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:22:42,856 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:22:42,857 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:22:42,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:42,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:42,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,858 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,43977,1733343762710, sessionid=0x100a6e635780000, setting cluster-up flag (Was=false) 2024-12-04T20:22:42,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,862 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:22:42,863 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,43977,1733343762710 2024-12-04T20:22:42,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:42,867 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:22:42,867 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,43977,1733343762710 2024-12-04T20:22:42,868 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:22:42,870 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:22:42,870 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:22:42,870 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:22:42,870 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,43977,1733343762710 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:22:42,871 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:22:42,872 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:22:42,872 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:22:42,872 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:22:42,872 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:22:42,872 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,872 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:22:42,872 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733343792874 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:22:42,874 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:22:42,874 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:22:42,874 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:22:42,875 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:22:42,875 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:22:42,875 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:22:42,875 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,875 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343762875,5,FailOnTimeoutGroup] 2024-12-04T20:22:42,875 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:22:42,875 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343762875,5,FailOnTimeoutGroup] 2024-12-04T20:22:42,875 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,875 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:22:42,875 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,875 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:22:42,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:22:42,880 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:22:42,881 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea 2024-12-04T20:22:42,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:22:42,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:22:42,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:42,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:22:42,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:22:42,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:42,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:22:42,889 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:22:42,889 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:42,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:22:42,890 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:22:42,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,890 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:42,890 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:22:42,891 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:22:42,891 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:42,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:42,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:22:42,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740 2024-12-04T20:22:42,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740 2024-12-04T20:22:42,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:22:42,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:22:42,894 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:22:42,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:22:42,896 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:22:42,896 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=847797, jitterRate=0.07802966237068176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:22:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733343762886Initializing all the Stores at 1733343762887 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343762887Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343762887Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343762887Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343762887Cleaning up temporary data from old regions at 1733343762893 (+6 ms)Region opened successfully at 1733343762897 (+4 ms) 2024-12-04T20:22:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:22:42,897 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:22:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:22:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:22:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:22:42,897 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:22:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343762897Disabling compacts and flushes for region at 1733343762897Disabling writes for close at 1733343762897Writing region close event to WAL at 1733343762897Closed at 1733343762897 2024-12-04T20:22:42,898 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:22:42,898 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:22:42,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:22:42,899 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:22:42,900 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:22:42,964 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(746): ClusterId : b77f3145-6f48-4b4a-b931-3029a9059325 2024-12-04T20:22:42,964 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:22:42,968 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:22:42,968 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:22:42,972 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:22:42,973 DEBUG [RS:0;c2ef38372881:41477 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aaf4385, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:22:42,987 DEBUG [RS:0;c2ef38372881:41477 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:41477 2024-12-04T20:22:42,987 INFO [RS:0;c2ef38372881:41477 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:22:42,987 INFO [RS:0;c2ef38372881:41477 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:22:42,987 DEBUG [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:22:42,988 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,43977,1733343762710 with port=41477, startcode=1733343762751 2024-12-04T20:22:42,988 DEBUG [RS:0;c2ef38372881:41477 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:22:42,990 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38541, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:22:42,990 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43977 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,41477,1733343762751 2024-12-04T20:22:42,990 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43977 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,41477,1733343762751 2024-12-04T20:22:42,992 DEBUG [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea 2024-12-04T20:22:42,992 DEBUG [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38243 2024-12-04T20:22:42,992 DEBUG [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:22:42,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:22:42,993 DEBUG [RS:0;c2ef38372881:41477 {}] zookeeper.ZKUtil(111): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,41477,1733343762751 2024-12-04T20:22:42,993 WARN [RS:0;c2ef38372881:41477 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:22:42,993 INFO [RS:0;c2ef38372881:41477 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:22:42,993 DEBUG [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/c2ef38372881,41477,1733343762751 2024-12-04T20:22:42,994 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,41477,1733343762751] 2024-12-04T20:22:42,996 INFO [RS:0;c2ef38372881:41477 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:22:42,997 INFO [RS:0;c2ef38372881:41477 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:22:42,998 INFO [RS:0;c2ef38372881:41477 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:22:42,998 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,998 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:22:42,998 INFO [RS:0;c2ef38372881:41477 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:22:42,998 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:22:42,999 DEBUG [RS:0;c2ef38372881:41477 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:22:42,999 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,999 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,999 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,999 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,999 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:42,999 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,41477,1733343762751-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:22:43,014 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:22:43,014 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,41477,1733343762751-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,014 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,014 INFO [RS:0;c2ef38372881:41477 {}] regionserver.Replication(171): c2ef38372881,41477,1733343762751 started 2024-12-04T20:22:43,026 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,026 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,41477,1733343762751, RpcServer on c2ef38372881/172.17.0.2:41477, sessionid=0x100a6e635780001 2024-12-04T20:22:43,026 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:22:43,026 DEBUG [RS:0;c2ef38372881:41477 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,41477,1733343762751 2024-12-04T20:22:43,026 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,41477,1733343762751' 2024-12-04T20:22:43,026 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:22:43,027 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:22:43,027 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:22:43,027 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:22:43,027 DEBUG [RS:0;c2ef38372881:41477 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,41477,1733343762751 2024-12-04T20:22:43,027 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,41477,1733343762751' 2024-12-04T20:22:43,027 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:22:43,028 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:22:43,028 DEBUG [RS:0;c2ef38372881:41477 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:22:43,028 INFO [RS:0;c2ef38372881:41477 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:22:43,028 INFO [RS:0;c2ef38372881:41477 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:22:43,050 WARN [c2ef38372881:43977 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T20:22:43,132 INFO [RS:0;c2ef38372881:41477 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C41477%2C1733343762751, suffix=, logDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/c2ef38372881,41477,1733343762751, archiveDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/oldWALs, maxLogs=32 2024-12-04T20:22:43,133 INFO [RS:0;c2ef38372881:41477 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C41477%2C1733343762751.1733343763133 2024-12-04T20:22:43,140 INFO [RS:0;c2ef38372881:41477 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/c2ef38372881,41477,1733343762751/c2ef38372881%2C41477%2C1733343762751.1733343763133 2024-12-04T20:22:43,141 DEBUG [RS:0;c2ef38372881:41477 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38423:38423),(127.0.0.1/127.0.0.1:44001:44001)] 2024-12-04T20:22:43,300 DEBUG [c2ef38372881:43977 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T20:22:43,301 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,41477,1733343762751 2024-12-04T20:22:43,304 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,41477,1733343762751, state=OPENING 2024-12-04T20:22:43,306 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:22:43,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:43,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:43,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:22:43,310 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:22:43,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:22:43,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,41477,1733343762751}] 2024-12-04T20:22:43,465 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:22:43,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60729, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:22:43,470 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:22:43,470 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:22:43,472 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C41477%2C1733343762751.meta, suffix=.meta, logDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/c2ef38372881,41477,1733343762751, archiveDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/oldWALs, maxLogs=32 2024-12-04T20:22:43,472 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c2ef38372881%2C41477%2C1733343762751.meta.1733343763472.meta 2024-12-04T20:22:43,479 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/c2ef38372881,41477,1733343762751/c2ef38372881%2C41477%2C1733343762751.meta.1733343763472.meta 2024-12-04T20:22:43,481 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38423:38423),(127.0.0.1/127.0.0.1:44001:44001)] 2024-12-04T20:22:43,489 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:22:43,489 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:22:43,489 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:22:43,489 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:22:43,490 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:22:43,490 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:22:43,490 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:22:43,490 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:22:43,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:22:43,492 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:22:43,492 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:43,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:43,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:22:43,493 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:22:43,493 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:43,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:43,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:22:43,494 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:22:43,494 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:43,494 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:43,494 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:22:43,495 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:22:43,495 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:22:43,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:22:43,495 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,37047,1733343592465/c2ef38372881%2C37047%2C1733343592465.1733343592641 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:43,495 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40063/user/jenkins/test-data/babb7652-990d-a12e-d088-ab88310eda4d/WALs/c2ef38372881,34471,1733343591112/c2ef38372881%2C34471%2C1733343591112.meta.1733343592355.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T20:22:43,495 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:22:43,496 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740 2024-12-04T20:22:43,497 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740 2024-12-04T20:22:43,498 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:22:43,498 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:22:43,498 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:22:43,499 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:22:43,499 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=764288, jitterRate=-0.02815866470336914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:22:43,499 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:22:43,500 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733343763490Writing region info on filesystem at 1733343763490Initializing all the Stores at 1733343763491 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343763491Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343763491Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343763491Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343763491Cleaning up temporary data from old regions at 1733343763498 (+7 ms)Running coprocessor post-open hooks at 1733343763499 (+1 ms)Region opened successfully at 1733343763500 (+1 ms) 2024-12-04T20:22:43,501 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733343763465 2024-12-04T20:22:43,503 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:22:43,503 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:22:43,504 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,41477,1733343762751 2024-12-04T20:22:43,504 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,41477,1733343762751, state=OPEN 2024-12-04T20:22:43,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:22:43,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:22:43,506 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,41477,1733343762751 2024-12-04T20:22:43,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:22:43,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:22:43,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:22:43,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,41477,1733343762751 in 196 msec 2024-12-04T20:22:43,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:22:43,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 611 msec 2024-12-04T20:22:43,512 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:22:43,512 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:22:43,513 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:22:43,513 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,41477,1733343762751, seqNum=-1] 2024-12-04T20:22:43,514 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:22:43,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55197, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:22:43,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 649 msec 2024-12-04T20:22:43,519 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733343763519, completionTime=-1 2024-12-04T20:22:43,519 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T20:22:43,520 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:22:43,521 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T20:22:43,521 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733343823521 2024-12-04T20:22:43,521 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733343883521 2024-12-04T20:22:43,521 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-04T20:22:43,522 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43977,1733343762710-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,522 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43977,1733343762710-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,522 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43977,1733343762710-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,522 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:43977, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,522 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,522 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,523 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:22:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.749sec 2024-12-04T20:22:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:22:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:22:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:22:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:22:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:22:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43977,1733343762710-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:22:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43977,1733343762710-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:22:43,527 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:22:43,527 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:22:43,527 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,43977,1733343762710-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:22:43,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5991282a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:22:43,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,43977,-1 for getting cluster id 2024-12-04T20:22:43,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:22:43,565 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b77f3145-6f48-4b4a-b931-3029a9059325' 2024-12-04T20:22:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:22:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b77f3145-6f48-4b4a-b931-3029a9059325" 2024-12-04T20:22:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703410e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:22:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,43977,-1] 2024-12-04T20:22:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:22:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:43,566 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:22:43,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371fd414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:22:43,567 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:22:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,41477,1733343762751, seqNum=-1] 2024-12-04T20:22:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:22:43,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:22:43,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,43977,1733343762710 2024-12-04T20:22:43,571 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:22:43,573 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T20:22:43,573 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T20:22:43,575 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/test.com,8080,1, archiveDir=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/oldWALs, maxLogs=32 2024-12-04T20:22:43,575 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733343763575 2024-12-04T20:22:43,580 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/test.com,8080,1/test.com%2C8080%2C1.1733343763575 2024-12-04T20:22:43,581 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44001:44001),(127.0.0.1/127.0.0.1:38423:38423)] 2024-12-04T20:22:43,581 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733343763581 2024-12-04T20:22:43,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,587 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/test.com,8080,1/test.com%2C8080%2C1.1733343763575 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/test.com,8080,1/test.com%2C8080%2C1.1733343763581 2024-12-04T20:22:43,588 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44001:44001),(127.0.0.1/127.0.0.1:38423:38423)] 2024-12-04T20:22:43,588 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/test.com,8080,1/test.com%2C8080%2C1.1733343763575 is not closed yet, will try archiving it next time 2024-12-04T20:22:43,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741835_1011 (size=93) 2024-12-04T20:22:43,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741835_1011 (size=93) 2024-12-04T20:22:43,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,590 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/WALs/test.com,8080,1/test.com%2C8080%2C1.1733343763575 to hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/oldWALs/test.com%2C8080%2C1.1733343763575 2024-12-04T20:22:43,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741836_1012 (size=93) 2024-12-04T20:22:43,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741836_1012 (size=93) 2024-12-04T20:22:43,593 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/oldWALs 2024-12-04T20:22:43,593 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733343763581) 2024-12-04T20:22:43,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:22:43,593 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:22:43,594 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:43,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:43,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:43,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:22:43,594 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:22:43,594 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=898795731, stopped=false 2024-12-04T20:22:43,594 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,43977,1733343762710 2024-12-04T20:22:43,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:43,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:43,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:22:43,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:22:43,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:43,595 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:22:43,595 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:43,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:43,595 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,41477,1733343762751' ***** 2024-12-04T20:22:43,595 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:22:43,596 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:22:43,596 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:22:43,596 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,41477,1733343762751 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:41477. 2024-12-04T20:22:43,596 DEBUG [RS:0;c2ef38372881:41477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:22:43,596 DEBUG [RS:0;c2ef38372881:41477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:22:43,596 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:22:43,597 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T20:22:43,597 DEBUG [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T20:22:43,597 DEBUG [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:22:43,597 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:22:43,597 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:22:43,597 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:22:43,597 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:22:43,597 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:22:43,597 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-12-04T20:22:43,612 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740/.tmp/ns/889fc12e4dc0470eae1a1c1345a23fae is 43, key is default/ns:d/1733343763515/Put/seqid=0 2024-12-04T20:22:43,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741837_1013 (size=5153) 2024-12-04T20:22:43,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741837_1013 (size=5153) 2024-12-04T20:22:43,617 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740/.tmp/ns/889fc12e4dc0470eae1a1c1345a23fae 2024-12-04T20:22:43,623 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740/.tmp/ns/889fc12e4dc0470eae1a1c1345a23fae as hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740/ns/889fc12e4dc0470eae1a1c1345a23fae 2024-12-04T20:22:43,626 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740/ns/889fc12e4dc0470eae1a1c1345a23fae, entries=2, sequenceid=6, filesize=5.0 K 2024-12-04T20:22:43,627 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false 2024-12-04T20:22:43,631 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T20:22:43,631 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:22:43,631 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:22:43,632 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343763597Running coprocessor pre-close hooks at 1733343763597Disabling compacts and flushes for region at 1733343763597Disabling writes for close at 1733343763597Obtaining lock to block concurrent updates at 1733343763597Preparing flush snapshotting stores in 1588230740 at 1733343763597Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733343763597Flushing stores of hbase:meta,,1.1588230740 at 1733343763598 (+1 ms)Flushing 1588230740/ns: creating writer at 1733343763598Flushing 1588230740/ns: appending metadata at 1733343763612 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733343763612Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eda9bab: reopening flushed file at 1733343763622 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false at 1733343763627 (+5 ms)Writing region close event to WAL at 1733343763628 (+1 ms)Running coprocessor post-close hooks at 1733343763631 (+3 ms)Closed at 1733343763631 2024-12-04T20:22:43,632 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:22:43,797 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,41477,1733343762751; all regions closed. 2024-12-04T20:22:43,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741834_1010 (size=1152) 2024-12-04T20:22:43,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741834_1010 (size=1152) 2024-12-04T20:22:43,804 DEBUG [RS:0;c2ef38372881:41477 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/oldWALs 2024-12-04T20:22:43,804 INFO [RS:0;c2ef38372881:41477 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C41477%2C1733343762751.meta:.meta(num 1733343763472) 2024-12-04T20:22:43,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741833_1009 (size=93) 2024-12-04T20:22:43,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741833_1009 (size=93) 2024-12-04T20:22:43,808 DEBUG [RS:0;c2ef38372881:41477 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/oldWALs 2024-12-04T20:22:43,808 INFO [RS:0;c2ef38372881:41477 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c2ef38372881%2C41477%2C1733343762751:(num 1733343763133) 2024-12-04T20:22:43,808 DEBUG [RS:0;c2ef38372881:41477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:22:43,808 INFO [RS:0;c2ef38372881:41477 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:22:43,808 INFO [RS:0;c2ef38372881:41477 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:22:43,809 INFO [RS:0;c2ef38372881:41477 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T20:22:43,809 INFO [RS:0;c2ef38372881:41477 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:22:43,809 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:22:43,809 INFO [RS:0;c2ef38372881:41477 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41477 2024-12-04T20:22:43,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:22:43,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,41477,1733343762751 2024-12-04T20:22:43,810 INFO [RS:0;c2ef38372881:41477 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:22:43,810 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,41477,1733343762751] 2024-12-04T20:22:43,811 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,41477,1733343762751 already deleted, retry=false 2024-12-04T20:22:43,811 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,41477,1733343762751 expired; onlineServers=0 2024-12-04T20:22:43,811 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,43977,1733343762710' ***** 2024-12-04T20:22:43,811 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:22:43,811 INFO [M:0;c2ef38372881:43977 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:22:43,811 INFO [M:0;c2ef38372881:43977 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:22:43,811 DEBUG [M:0;c2ef38372881:43977 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:22:43,811 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:22:43,811 DEBUG [M:0;c2ef38372881:43977 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:22:43,811 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343762875 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343762875,5,FailOnTimeoutGroup] 2024-12-04T20:22:43,811 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343762875 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343762875,5,FailOnTimeoutGroup] 2024-12-04T20:22:43,812 INFO [M:0;c2ef38372881:43977 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:22:43,812 INFO [M:0;c2ef38372881:43977 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:22:43,812 DEBUG [M:0;c2ef38372881:43977 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:22:43,812 INFO [M:0;c2ef38372881:43977 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:22:43,812 INFO [M:0;c2ef38372881:43977 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:22:43,812 INFO [M:0;c2ef38372881:43977 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:22:43,812 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:22:43,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:22:43,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:22:43,812 DEBUG [M:0;c2ef38372881:43977 {}] zookeeper.ZKUtil(347): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:22:43,812 WARN [M:0;c2ef38372881:43977 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:22:43,813 INFO [M:0;c2ef38372881:43977 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/.lastflushedseqids 2024-12-04T20:22:43,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741838_1014 (size=99) 2024-12-04T20:22:43,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741838_1014 (size=99) 2024-12-04T20:22:43,818 INFO [M:0;c2ef38372881:43977 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:22:43,818 INFO [M:0;c2ef38372881:43977 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:22:43,818 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:22:43,818 INFO [M:0;c2ef38372881:43977 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:43,818 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:43,818 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:22:43,818 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:43,818 INFO [M:0;c2ef38372881:43977 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-12-04T20:22:43,832 DEBUG [M:0;c2ef38372881:43977 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/59b7cb5bf86345278672d6cc9bb35285 is 82, key is hbase:meta,,1/info:regioninfo/1733343763503/Put/seqid=0 2024-12-04T20:22:43,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741839_1015 (size=5672) 2024-12-04T20:22:43,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741839_1015 (size=5672) 2024-12-04T20:22:43,837 INFO [M:0;c2ef38372881:43977 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/59b7cb5bf86345278672d6cc9bb35285 2024-12-04T20:22:43,853 DEBUG [M:0;c2ef38372881:43977 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb062c9dba8548e79614cc3a259a7fcd is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733343763519/Put/seqid=0 2024-12-04T20:22:43,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741840_1016 (size=5275) 2024-12-04T20:22:43,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741840_1016 (size=5275) 2024-12-04T20:22:43,858 INFO [M:0;c2ef38372881:43977 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb062c9dba8548e79614cc3a259a7fcd 2024-12-04T20:22:43,874 DEBUG [M:0;c2ef38372881:43977 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6f70cc74fd34ac580d594ce2076667f is 69, key is c2ef38372881,41477,1733343762751/rs:state/1733343762991/Put/seqid=0 2024-12-04T20:22:43,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741841_1017 (size=5156) 2024-12-04T20:22:43,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741841_1017 (size=5156) 2024-12-04T20:22:43,878 INFO [M:0;c2ef38372881:43977 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6f70cc74fd34ac580d594ce2076667f 2024-12-04T20:22:43,896 DEBUG [M:0;c2ef38372881:43977 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3b52693b090b463aaf59cddb40d8274c is 52, key is load_balancer_on/state:d/1733343763572/Put/seqid=0 2024-12-04T20:22:43,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741842_1018 (size=5056) 2024-12-04T20:22:43,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741842_1018 (size=5056) 2024-12-04T20:22:43,901 INFO [M:0;c2ef38372881:43977 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3b52693b090b463aaf59cddb40d8274c 2024-12-04T20:22:43,905 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/59b7cb5bf86345278672d6cc9bb35285 as hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/59b7cb5bf86345278672d6cc9bb35285 2024-12-04T20:22:43,908 INFO [M:0;c2ef38372881:43977 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/59b7cb5bf86345278672d6cc9bb35285, entries=8, sequenceid=29, filesize=5.5 K 2024-12-04T20:22:43,909 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb062c9dba8548e79614cc3a259a7fcd as hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bb062c9dba8548e79614cc3a259a7fcd 2024-12-04T20:22:43,911 INFO [RS:0;c2ef38372881:41477 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:22:43,911 INFO [RS:0;c2ef38372881:41477 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,41477,1733343762751; zookeeper connection closed. 2024-12-04T20:22:43,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:43,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41477-0x100a6e635780001, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:43,911 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2c4b1464 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2c4b1464 2024-12-04T20:22:43,911 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T20:22:43,912 INFO [M:0;c2ef38372881:43977 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bb062c9dba8548e79614cc3a259a7fcd, entries=3, sequenceid=29, filesize=5.2 K 2024-12-04T20:22:43,913 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6f70cc74fd34ac580d594ce2076667f as hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a6f70cc74fd34ac580d594ce2076667f 2024-12-04T20:22:43,917 INFO [M:0;c2ef38372881:43977 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a6f70cc74fd34ac580d594ce2076667f, entries=1, sequenceid=29, filesize=5.0 K 2024-12-04T20:22:43,918 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3b52693b090b463aaf59cddb40d8274c as hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3b52693b090b463aaf59cddb40d8274c 2024-12-04T20:22:43,921 INFO [M:0;c2ef38372881:43977 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38243/user/jenkins/test-data/7f4a88fb-3f51-a338-0ec2-845b38cd3bea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3b52693b090b463aaf59cddb40d8274c, entries=1, sequenceid=29, filesize=4.9 K 2024-12-04T20:22:43,922 INFO [M:0;c2ef38372881:43977 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=29, compaction requested=false 2024-12-04T20:22:43,924 INFO [M:0;c2ef38372881:43977 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:22:43,924 DEBUG [M:0;c2ef38372881:43977 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343763818Disabling compacts and flushes for region at 1733343763818Disabling writes for close at 1733343763818Obtaining lock to block concurrent updates at 1733343763818Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733343763818Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733343763819 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733343763819Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733343763819Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733343763832 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733343763832Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733343763840 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733343763853 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733343763853Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733343763861 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733343763873 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733343763873Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733343763881 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733343763896 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733343763896Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d77079f: reopening flushed file at 1733343763904 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d1abb21: reopening flushed file at 1733343763908 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62b20808: reopening flushed file at 1733343763912 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45446e53: reopening flushed file at 1733343763917 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=29, compaction requested=false at 1733343763922 (+5 ms)Writing region close event to WAL at 1733343763923 (+1 ms)Closed at 1733343763923 2024-12-04T20:22:43,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:22:43,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39815 is added to blk_1073741830_1006 (size=10311) 2024-12-04T20:22:43,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33475 is added to blk_1073741830_1006 (size=10311) 2024-12-04T20:22:43,929 INFO [M:0;c2ef38372881:43977 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:22:43,929 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:22:43,929 INFO [M:0;c2ef38372881:43977 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43977 2024-12-04T20:22:43,930 INFO [M:0;c2ef38372881:43977 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:22:44,031 INFO [M:0;c2ef38372881:43977 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:22:44,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:44,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43977-0x100a6e635780000, quorum=127.0.0.1:54065, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:22:44,034 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@413a6699{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:44,035 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1cfa6b2d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:44,035 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:44,035 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2578bc63{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:44,035 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e818af2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:44,036 WARN [BP-645053777-172.17.0.2-1733343762179 heartbeating to localhost/127.0.0.1:38243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:22:44,037 WARN [BP-645053777-172.17.0.2-1733343762179 heartbeating to localhost/127.0.0.1:38243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-645053777-172.17.0.2-1733343762179 (Datanode Uuid 1f39bd39-b261-49f3-97a3-badc5e6c22b2) service to localhost/127.0.0.1:38243 2024-12-04T20:22:44,037 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:22:44,037 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:22:44,037 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/data/data3/current/BP-645053777-172.17.0.2-1733343762179 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:44,037 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/data/data4/current/BP-645053777-172.17.0.2-1733343762179 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:44,038 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:22:44,040 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4578d3a2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:22:44,040 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@259cffcc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:44,040 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:44,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d148abe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:44,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20042785{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:44,042 WARN [BP-645053777-172.17.0.2-1733343762179 heartbeating to localhost/127.0.0.1:38243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:22:44,042 WARN [BP-645053777-172.17.0.2-1733343762179 heartbeating to localhost/127.0.0.1:38243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-645053777-172.17.0.2-1733343762179 (Datanode Uuid b61205d4-cef8-4a03-ac4c-ba7d82def94d) service to localhost/127.0.0.1:38243 2024-12-04T20:22:44,042 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:22:44,042 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:22:44,043 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/data/data1/current/BP-645053777-172.17.0.2-1733343762179 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:44,043 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/cluster_cb35c6c6-84f9-e41e-4b48-61794df48649/data/data2/current/BP-645053777-172.17.0.2-1733343762179 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:22:44,043 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:22:44,048 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10c1adfc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:22:44,049 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5fab3ee3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:22:44,049 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:22:44,049 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ee125cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:22:44,049 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a024a47{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df37e76d-7854-5967-4a70-b557945f9eb2/hadoop.log.dir/,STOPPED} 2024-12-04T20:22:44,055 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:22:44,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T20:22:44,081 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=283 (was 233) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38243 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38243 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38243 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38243 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38243 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38243 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38243 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:38243 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=561 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=78 (was 78), ProcessCount=11 (was 11), AvailableMemoryMB=4712 (was 4729)